Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
534/// on an early return or a panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Recorded before the first save, so a crash right after minting
773        // cannot leave a run with no origin. Legibility only: nothing reads it
774        // to decide anything.
775        state.fetched_base = Some(fetched_base);
776        state.origin = Some(origin);
777        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
778        state.status = RunStatus::Prep;
779        state.event("start", format!("run {} created", state.id));
780        startup.finish(&mut state, Ok(()))?;
781        Ok(Self {
782            state,
783            roles,
784            sem: Arc::new(Semaphore::new(max_parallel)),
785            pause: Pause::new(),
786            interrupt: Pause::new(),
787        })
788    }
789
790    /// Open a review-only run against work that already exists on `branch`.
791    ///
792    /// The expensive half of the graph is the implement wave — measured at
793    /// 111 and 134 internal tool-loop turns on this repository, against a
794    /// handful for a judge or a reviewer. The cheap half is worth running on
795    /// hand-written work too, and there was no way to reach it.
796    ///
797    /// No new state and no schema change are needed: a run with **one** viable
798    /// candidate and a tally already decided degrades `execute` to exactly
799    /// review → gate → merge, because `judge` skips a single-candidate field,
800    /// `deliberate` has fewer than two first choices to reconcile, `vote`
801    /// returns early, `tally` is already present and `fold_losers` has no
802    /// losers. Resuming such a run therefore does the right thing as well.
803    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
804        Self::review_taking_over(repo, branch, config, None, origin).await
805    }
806
807    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
808    /// at the same task still has `branch` checked out, its worktree is
809    /// released first if that is safe (see [`crate::handover`]), and the
810    /// review refuses with the reason if it is not. `None` is a hand-run
811    /// review: it has no earlier attempts, so only a worktree of a dead run
812    /// magi recorded itself can be released.
813    pub async fn review_taking_over(
814        repo: &Path,
815        branch: &str,
816        config: Config,
817        takeover: Option<crate::handover::Takeover>,
818        origin: Origin,
819    ) -> Result<Self> {
820        let repo = git::toplevel(repo).await?;
821        let missing = agent::missing_programs(&config.agents);
822        if !missing.is_empty() {
823            bail!(
824                "these agent programs are not on PATH: {}. Fix the roster in \
825                 magi.toml or install them.",
826                missing.join(", ")
827            );
828        }
829        let base_branch =
830            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
831                .await?;
832        if base_branch == branch {
833            bail!("`{branch}` is the base branch; there is nothing to review against");
834        }
835        let fetched_base = resolve_base(
836            &repo,
837            &base_branch,
838            &config.merge.remote,
839            config.coordination.as_ref(),
840        )
841        .await?;
842        let base_commit = fetched_base.commit.clone();
843
844        let roles = config.resolve_roles()?;
845        let max_parallel = config.graph.max_parallel.max(1);
846        let mut state = RunState::new(
847            repo.clone(),
848            base_branch,
849            base_commit.clone(),
850            format!("Review the work already on branch `{branch}`."),
851            config,
852        );
853        state.fetched_base = Some(fetched_base);
854        state.origin = Some(origin);
855        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
856        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
857        let opened = async {
858            startup.stage(&mut state, "handover")?;
859
860            // Released before anything else touches the branch: a stale local
861            // branch is moved with `git branch -f`, which git refuses while an
862            // earlier attempt's worktree still has it checked out. Everything
863            // after this point that can fail puts the old run back.
864            // A hand-run review has no task, hence no earlier attempts, but a
865            // worktree of a dead run magi made may still be released.
866            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
867                earlier: Vec::new(),
868                home: crate::run::home(),
869                choice: None,
870                task_instruction: None,
871            });
872            let wait = repository_wait(&state, None);
873            let coordination_config = state.config.coordination.clone();
874            let coordination =
875                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
876                    .await?;
877            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
878            if let Some(released) = &released {
879                state.event(
880                    "release",
881                    format!(
882                        "took `{branch}` over from run {}: its worktree was released: {}",
883                        crate::run::short_of(&released.old_id),
884                        released.audit
885                    ),
886                );
887            }
888            let opened = async {
889                startup.stage(&mut state, "review-preflight")?;
890                // The owner's answer to an earlier divergence question is applied
891                // here: after the release (git will not move a checked-out branch)
892                // and before the sync that would otherwise ask again.
893                state.foreign_checkout = crate::handover::preflight_review(
894                    &repo,
895                    branch,
896                    &takeover.home,
897                    &state.config.merge.remote,
898                )
899                .await?;
900                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
901                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
902                }
903                if let Some(choice) = takeover.choice.as_ref()
904                    && let Err(e) =
905                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
906                            .await
907                {
908                    return Err(e.context("applying the owner's answer about the diverged branch"));
909                }
910                // An earlier attempt of this task that published the task head, and
911                // whose work is the branch under review, hands that head over: the
912                // review run updates the same pull request. Without one (a legacy
913                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
914                // own head and its own open pull request is adopted as before.
915                for id in takeover.earlier.iter().rev() {
916                    if let Ok(old) = RunState::load_under(id, &takeover.home)
917                        && let Some(head) = old.pr_head.clone()
918                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
919                    {
920                        state.pr_head = Some(head);
921                        break;
922                    }
923                }
924                drop(coordination);
925                startup.stage(&mut state, "open-review")?;
926                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
927            }
928            .await;
929            if opened.is_err()
930                && let Some(released) = &released
931            {
932                // Restoration is ownership checked and best effort. Keep the original
933                // startup cause even if reacquiring coordination is no longer possible.
934                match crate::repo_lock::SeatLease::operation_with(
935                    &repo,
936                    coordination_config.as_ref(),
937                    &wait,
938                )
939                .await
940                {
941                    Ok(_coordination) => {
942                        released.restore(&repo, branch).await;
943                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
944                            .map(|old| {
945                                format!(
946                                    "predecessor released_to={:?}; candidate path exists={}",
947                                    old.released_to,
948                                    old.candidates.iter().any(|c| {
949                                        c.branch == branch && c.worktree.exists()
950                                    }),
951                                )
952                            })
953                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
954                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
955                    }
956                    Err(error) => {
957                        tracing::warn!(
958                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
959                            state.id
960                        );
961                        state.event(
962                            "startup-restore",
963                            format!("restoration not attempted: {error:#}"),
964                        );
965                    }
966                }
967            }
968            opened
969        }
970        .await;
971        startup.finish(&mut state, opened)?;
972        Ok(Self {
973            state,
974            roles,
975            sem: Arc::new(Semaphore::new(max_parallel)),
976            pause: Pause::new(),
977            interrupt: Pause::new(),
978        })
979    }
980
981    /// Recover exactly the operator-selected public implementation, without touching old sources.
982    pub async fn recover_pr(
983        repo: &Path,
984        task: &str,
985        selection: crate::pr_recovery::Selection,
986        mut config: Config,
987    ) -> Result<Self> {
988        selection.configure(&mut config)?;
989        selection.check_task(task, &selection.instruction)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.origin = Some(crate::run::Origin::queue(task));
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize exactly one additional rebase, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true).await
1357    }
1358
1359    async fn authorize_base_recovery_request(
1360        home: &Path,
1361        id: &str,
1362        task: &str,
1363        resumption: bool,
1364    ) -> Result<()> {
1365        let _owner = crate::pr_head::owner_lock(home, id)?;
1366        let mut state = RunState::load_under(id, home)?;
1367        crate::pr_head::ensure_owner(&state)?;
1368        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369        anyhow::ensure!(
1370            queued.runs.last().is_some_and(|run| run == id)
1371                && queued.status != crate::queue::TaskStatus::Done
1372                && !queued.fresh_start
1373                && queued.review_branch.is_none(),
1374            "run task ownership changed"
1375        );
1376        anyhow::ensure!(
1377            state
1378                .origin
1379                .as_ref()
1380                .and_then(|o| o.task.as_deref())
1381                .is_none_or(|owner| owner == task),
1382            "run belongs to another task"
1383        );
1384        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385        anyhow::ensure!(
1386            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387            "run is live or its ownership is unknown"
1388        );
1389        // Judge the previous driver's lease before acquiring our maintenance
1390        // lease. The owner lock excludes new driver claims across both steps.
1391        let _execution = crate::driver::maintenance(home, id)?;
1392        anyhow::ensure!(
1393            state.status.resumable()
1394                && state.released_to.is_none()
1395                && state.retired_to.is_none()
1396                && state.pr_taken_over_by.is_none(),
1397            "run ownership is final"
1398        );
1399        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400        anyhow::ensure!(
1401            !state.pr_publication_pending
1402                && !crate::handover::landing_checkpoint(&state)
1403                && state.merge_decision.is_none(),
1404            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405        );
1406        let winner = state.winner().context("run has no retained winner")?;
1407        anyhow::ensure!(
1408            git::is_clean(&winner.worktree).await?,
1409            "retained worktree has uncommitted changes"
1410        );
1411        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412        anyhow::ensure!(
1413            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414            "retained worktree does not match the winner branch"
1415        );
1416        state.record_base_sync_stop(&head);
1417        state.save()?;
1418        if resumption {
1419            anyhow::ensure!(
1420                crate::daemon::compatible_run_instruction(&state, &queued),
1421                "resumption instruction does not match the retained run"
1422            );
1423            let approval = queued
1424                .resumption
1425                .as_ref()
1426                .context("no operator resumption")?;
1427            anyhow::ensure!(
1428                approval.instruction == queued.instruction,
1429                "resumption approval does not match the current task instruction"
1430            );
1431            state.authorize_base_sync_resumption(task, approval)?;
1432        } else {
1433            state.authorize_base_sync(task)?;
1434        }
1435        state.event(
1436            "base-recovery",
1437            "owner authorized one additional base rebase; fresh verification required",
1438        );
1439        state.save()?;
1440        Ok(())
1441    }
1442
1443    /// Reopen an existing run.
1444    pub fn resume(id: &str) -> Result<Self> {
1445        let state = RunState::load(id)?;
1446        if state.startup.as_ref().is_some_and(|s| !s.complete)
1447            || (state.status == RunStatus::Prep
1448                && state.instruction.trim().is_empty()
1449                && state.candidates.is_empty()
1450                && state.prep_complete == Some(false))
1451        {
1452            bail!(
1453                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1454            );
1455        }
1456        crate::pr_head::ensure_owner(&state)?;
1457        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1458            bail!(
1459                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1460            );
1461        }
1462        if let Some(to) = &state.released_to {
1463            bail!(
1464                "run {} cannot be resumed: its worktree was released to run {}",
1465                state.short(),
1466                crate::run::short_of(to)
1467            );
1468        }
1469        let roles = state.config.resolve_roles()?;
1470        if let Some(id) = &state.config.graph.selected_implementer
1471            && state.candidates.iter().any(|c| c.agent != *id)
1472        {
1473            bail!(
1474                "replacement implementer pin does not match the saved candidates; task must stay held"
1475            );
1476        }
1477        let max_parallel = state.config.graph.max_parallel.max(1);
1478        Ok(Self {
1479            state,
1480            roles,
1481            sem: Arc::new(Semaphore::new(max_parallel)),
1482            pause: Pause::new(),
1483            interrupt: Pause::new(),
1484        })
1485    }
1486
1487    /// Walk the graph to a terminal state, skipping nodes already recorded.
1488    ///
1489    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1490    /// the phone - ends here, so this is the one place a run that ended
1491    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1492    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1493    pub async fn execute(&mut self) -> Result<()> {
1494        anyhow::ensure!(
1495            self.state.startup.as_ref().is_none_or(|s| s.complete),
1496            "startup incomplete; inspect retained evidence before explicit recovery"
1497        );
1498        if let Some(selection) = &self.state.pr_recovery
1499            && (selection.source_proof.is_some() || selection.source_run.is_some())
1500        {
1501            Box::pin(selection.verify_source(&self.state.repo)).await?;
1502        }
1503        // Queue claims cover daemon work; this also serializes manual drivers
1504        // consuming a repair answer, including the new review and decision.
1505        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1506            || self.state.land_approval.is_some()
1507            || self.state.merge_decision.is_some()
1508            || !self.state.merge_repairs.is_empty()
1509            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1510        {
1511            let claim = FixClaim::acquire(&self.state.dir())?;
1512            let disk = RunState::load(&self.state.id)?;
1513            let checkpoint = |s: &RunState| {
1514                (
1515                    s.merge_repairs.len(),
1516                    s.merge_repairs
1517                        .last()
1518                        .map(|r| (r.question.clone(), r.phase)),
1519                    s.land_approval
1520                        .as_ref()
1521                        .map(|a| (a.question.clone(), a.head.clone())),
1522                )
1523            };
1524            if checkpoint(&disk) != checkpoint(&self.state) {
1525                bail!(
1526                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1527                );
1528            }
1529            Some(claim)
1530        } else {
1531            None
1532        };
1533        let mut execution = self.claim_driver().await?;
1534        let runner = &mut *execution.runner;
1535        let wait = runner.repository_wait();
1536        // Keep the graph's large future off the callers' stack, including
1537        // standalone task attempts running on a default-sized test thread.
1538        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1539        let result = match result {
1540            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1541                record_repository_block(&mut runner.state, &e);
1542                runner.park_here()?;
1543                Ok(())
1544            }
1545            other => other,
1546        };
1547        execution.driver.finish(&mut runner.state).await?;
1548        let ended = if result.is_err() {
1549            Some({
1550                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1551                if let Err(error) = &result {
1552                    notice.message = format!(
1553                        "Run {} stopped with an error: {}",
1554                        runner.state.id,
1555                        error.to_string().lines().next().unwrap_or("unknown error")
1556                    );
1557                }
1558                notice
1559            })
1560        } else {
1561            crate::notices::run_ended(&runner.state)
1562        };
1563        if let Some(notice) = ended {
1564            crate::notices::raise(notice);
1565        }
1566        result
1567    }
1568
1569    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1570        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1571    }
1572
1573    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1574        let wait = self.repository_wait();
1575        coordinate_operation(&mut self.state, cwd, &wait).await
1576    }
1577
1578    /// Validate ownership and persist the driver claim in one critical section.
1579    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1580        let owner = match crate::pr_head::owner_lock_wait(
1581            &crate::run::home(),
1582            &self.state.id,
1583            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1584        )
1585        .await
1586        {
1587            Ok(owner) => owner,
1588            Err(error) => {
1589                let detail = format!("worktree maintenance or ownership update: {error:#}");
1590                // Admission did not own run.json: retain this resource outcome
1591                // in memory for queue settlement without overwriting a concurrent
1592                // retirement or publication transaction.
1593                self.state.resource_block = Some(crate::run::ResourceBlock {
1594                    cause: crate::run::ResourceBlockCause::Other,
1595                    node: "start".into(),
1596                    kind: "worktree maintenance".into(),
1597                    detail: detail.clone(),
1598                    at: Timestamp::now(),
1599                });
1600                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1601            }
1602        };
1603        let disk = RunState::load(&self.state.id)?;
1604        crate::pr_head::ensure_owner(&disk)?;
1605        let claimed =
1606            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1607        // Only proven-live ownership refuses here. Unknown records (legacy runs
1608        // with no claim marker or process identity) fall through to the
1609        // exclusive execution lease taken just below, which is what actually
1610        // keeps a second driver out.
1611        anyhow::ensure!(
1612            disk.liveness(claimed) != crate::run::Liveness::Live,
1613            "run already has a live driver; wait for its execution to end"
1614        );
1615        if let Some(to) = &disk.released_to {
1616            bail!(
1617                "run {} cannot continue: its worktree was released to run {}",
1618                self.state.short(),
1619                crate::run::short_of(to)
1620            );
1621        }
1622        anyhow::ensure!(
1623            serde_json::to_value(&disk.base_sync_recovery)?
1624                == serde_json::to_value(&self.state.base_sync_recovery)?,
1625            "base recovery changed since this run was loaded; reload the saved run"
1626        );
1627        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1628        let mut execution = DriverExecution {
1629            runner: self,
1630            driver,
1631        };
1632        // Declare after the execution guard: on any claim error this short
1633        // lock must drop before that guard reloads and releases ownership.
1634        let _owner = owner;
1635        let state = &mut execution.runner.state;
1636        // Moving again, so it is no longer parked. Set before the walk rather
1637        // than in `resume`, so every way of re-entering the graph clears it
1638        // and a card cannot claim a run is waiting to be resumed while the
1639        // agents are already working.
1640        state.parked = false;
1641        state.resource_wait = None;
1642        state.resource_block = None;
1643        // Any seat this state still lists as answering belongs to whatever
1644        // process last drove this run — this one included, if it crashed
1645        // mid-wave. Cleared and flushed immediately, before anything else
1646        // runs, so a resume can never show a seat as live when nothing is
1647        // asking it anything yet; the node that actually dispatches the next
1648        // wave repopulates it.
1649        state.clear_active();
1650        // Recorded in the same spot, and flushed together with the clear
1651        // above: this is the pid a reader checks (`RunState::liveness`) when
1652        // no daemon claim exists to answer "is a process still driving this
1653        // run" — a plain `magi run` / `magi review` typed into a terminal
1654        // claims nothing there. Always overwritten, never only-if-absent, so
1655        // a resumed run's stale pid from a previous, possibly-dead process
1656        // can never survive into this one's own report. Unlike
1657        // `clear_active`, this changes on every single `execute()` call, so
1658        // the save below is now unconditional rather than only-if-cleared.
1659        //
1660        // `driver_started_at` is recorded in the same breath, from this same
1661        // pid, so `liveness` can tell a live pid that is genuinely still us
1662        // apart from one the OS has since handed to an unrelated process —
1663        // see that field's own doc for why the pid alone is not enough.
1664        let pid = std::process::id();
1665        state.driver_pid = Some(pid);
1666        state.driver_started_at = crate::proc::process_started_at(pid);
1667        state.driver_exited = false;
1668        state.driver_claimed = Some(true);
1669        state.begin_pass();
1670        execution.driver.bind(state);
1671        state.save()?;
1672        execution.driver.record(state)?;
1673        Ok(execution)
1674    }
1675
1676    async fn execute_graph(&mut self) -> Result<()> {
1677        self.state.configure_recovery_resume()?;
1678        // Preservation may invalidate an old empty classification. Resolve it
1679        // before any resume shortcut can reuse a verdict or publish a branch.
1680        self.persist_candidates().await?;
1681        // A saved base stop outranks stage markers, including legacy Voting.
1682        // Recovery never reenters implementation, deliberation or voting.
1683        if self.state.base_sync_blocked() {
1684            self.sync_to_base().await?;
1685            if !self.state.base_sync_blocked()
1686                && self.state.status != RunStatus::AlreadyInBase
1687                && !self.resource_halt()?
1688            {
1689                self.finish_after_tally().await?;
1690            }
1691            return Ok(());
1692        }
1693        if self
1694            .state
1695            .merge_decision
1696            .as_ref()
1697            .is_some_and(|decision| !decision.closed)
1698            && self.state.status != RunStatus::Landing
1699        {
1700            if crate::merge_decision::resumable_hold(&self.state) {
1701                self.state.merge = Some(MergeOutcome {
1702                    mode: MergeMode::Pr,
1703                    ok: false,
1704                    detail: self
1705                        .state
1706                        .merge_decision
1707                        .as_ref()
1708                        .unwrap()
1709                        .target
1710                        .pr
1711                        .clone(),
1712                    empty: false,
1713                });
1714                self.run_land().await?;
1715                self.settle_questions();
1716                return Ok(());
1717            }
1718            bail!(
1719                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1720            );
1721        }
1722        if self.state.config.graph.selected_implementer.is_some()
1723            && self.state.status == RunStatus::Blocked
1724            && self.state.tally.is_none()
1725            && self.state.candidate_preservation.is_empty()
1726            && self.state.interrupted_implementations.is_empty()
1727        {
1728            bail!(
1729                "selected implementation is held; explicitly release the owning task to retry it"
1730            );
1731        }
1732
1733        if self.state.merge_repairs.last().is_some_and(|r| {
1734            matches!(
1735                r.phase,
1736                crate::run::MergeRepairPhase::Requested
1737                    | crate::run::MergeRepairPhase::Fixing
1738                    | crate::run::MergeRepairPhase::Reviewing
1739            )
1740        }) {
1741            if self.repair_merge_review().await? {
1742                self.run_land().await?;
1743            }
1744            self.settle_questions();
1745            return Ok(());
1746        }
1747        // Publication already passed review and gate. Resume only that node:
1748        // syncing to a newer base here could change the verified commit.
1749        if self.state.pr_publication_pending {
1750            self.merge().await?;
1751            if self.resource_halt()? {
1752                return Ok(());
1753            }
1754            self.state.save()?;
1755            self.settle_questions();
1756            return Ok(());
1757        }
1758        // A run that already lost its quorum never resumes into the verdict
1759        // machinery: `deliberate` and `vote` would otherwise clobber the
1760        // stalled marker back to Voting and the run would keep going past a
1761        // verdict that is no longer trustworthy. Everything already recorded is
1762        // kept, so the run stays resumable (or foldable) for a human to pick up.
1763        //
1764        // On --resume the run gets one chance to repair itself: the seats a
1765        // rate limit took out are re-asked. If their quota has since reset and
1766        // the quorum is restored, the run picks up and finishes; otherwise it
1767        // stays stale and still-resumable for a later retry. If it does not
1768        // recover, the returned status stays `Stalled` and nothing was
1769        // clobbered (the recovery only mutates entries for the lost seats).
1770        if self.state.status == RunStatus::Stalled {
1771            if self.recover_stall().await? {
1772                self.finish_after_tally().await?;
1773            } else {
1774                // Still below quorum: persist the marker and stay resumable.
1775                self.state.save()?;
1776            }
1777            return Ok(());
1778        }
1779        // A run parked inside `land` - watching CI, mid fix-round, or
1780        // waiting on the owner's merge approval - resumes directly into it,
1781        // never back through `prep`. Everything before `merge` already
1782        // concluded; that is the only way `status` reaches `Landing` in the
1783        // first place. Re-walking `review_loop` first would also be actively
1784        // wrong: its own status recomputation (see its doc) treats any
1785        // clean round as reason to set `status` to `Gating`, which would
1786        // clobber this marker before `merge` ever ran, and this run would
1787        // never find its way back into `land` at all.
1788        if self.state.status == RunStatus::Landing
1789            || crate::handover::landing_checkpoint(&self.state)
1790        {
1791            self.state.land_sync_pending = true;
1792            self.state.save()?;
1793            self.run_land().await?;
1794            // `run_land` may have settled the run right here - CI came back
1795            // green and the PR merged, say - without ever passing back
1796            // through `merge`'s own trailing call. Whatever it left `status`
1797            // as is what this has to read.
1798            self.settle_questions();
1799            return Ok(());
1800        }
1801        // A run parked at `merge` on the owner's word about a withheld PR text
1802        // resumes straight into `merge`. Re-walking the tail would sync to the
1803        // base again (possibly rebasing, possibly spending the fixer) after the
1804        // gate had passed, and then push a tree nobody reviewed or gated.
1805        if self.state.github_text.is_some()
1806            && self.state.merge.is_none()
1807            && self.state.status == RunStatus::Gating
1808        {
1809            self.merge().await?;
1810            if self.resource_halt()? {
1811                return Ok(());
1812            }
1813            self.state.save()?;
1814            return Ok(());
1815        }
1816        if let Err(e) = self.prep().await {
1817            if e.is::<crate::repo_lock::ResourceBlocked>() {
1818                return Err(e);
1819            }
1820            self.state.status = RunStatus::Blocked;
1821            self.state
1822                .prep_failure
1823                .get_or_insert_with(|| format!("{e:#}"));
1824            self.state
1825                .event("prep", format!("preparation stopped: {e:#}"));
1826            self.state.save()?;
1827            return Err(e);
1828        }
1829        if self.park_here()? {
1830            return Ok(());
1831        }
1832        self.advise().await?;
1833        if self.park_here()? {
1834            return Ok(());
1835        }
1836        self.implement().await?;
1837        if self.park_here()? {
1838            return Ok(());
1839        }
1840        // `after_implement` already saved the state and settled any open
1841        // questions when it set this; nothing later in the graph has
1842        // anything to judge.
1843        if self.state.status == RunStatus::VerifiedNoop {
1844            return Ok(());
1845        }
1846        self.judge().await?;
1847        if self.park_here()? {
1848            return Ok(());
1849        }
1850        self.deliberate().await?;
1851        if self.park_here()? {
1852            return Ok(());
1853        }
1854        self.vote().await?;
1855        if self.park_here()? {
1856            return Ok(());
1857        }
1858        self.tally()?;
1859        // A verdict that lost its quorum is not trustworthy: do not review,
1860        // gate, or merge on it. Everything already done is kept, so the run
1861        // stays resumable (or foldable); the human can replace the agent that
1862        // ran out of quota and pick it up.
1863        if self.state.status == RunStatus::Stalled {
1864            // Persist the stalled marker now — the normal end-of-execute save
1865            // below is below this early return, and without it a resumed run
1866            // would reload a pre-tally status and keep going.
1867            self.state.save()?;
1868            return Ok(());
1869        }
1870        self.finish_after_tally().await?;
1871        Ok(())
1872    }
1873
1874    /// A wave could not start its agents because a shared resource stayed busy
1875    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1876    /// resumable. Unstarted seats have no result; completed peers retain
1877    /// their saved candidate state.
1878    fn stop_for_resource(&mut self) -> Result<()> {
1879        // Resource acquisition observes cancellation inside a node. Complete
1880        // the same durable park as every other node boundary before returning.
1881        if self.pause.parked() || self.interrupt.parked() {
1882            self.park_here()?;
1883            return Ok(());
1884        }
1885        if self.state.resource_block.is_some() {
1886            self.state.status = RunStatus::Blocked;
1887        }
1888        self.state.save()
1889    }
1890
1891    /// Stop walking the graph if a node ended on a resource block.
1892    fn resource_halt(&mut self) -> Result<bool> {
1893        if self.state.resource_block.is_none() {
1894            return Ok(false);
1895        }
1896        self.stop_for_resource()?;
1897        Ok(true)
1898    }
1899
1900    /// Park here if asked to, recording it in the run's own timeline.
1901    ///
1902    /// Returns whether the caller should stop walking the graph. The state is
1903    /// saved either way by the node that just finished; this adds the event so
1904    /// the operator's card says why a run that is neither finished nor moving
1905    /// is sitting where it is.
1906    fn park_here(&mut self) -> Result<bool> {
1907        let parking = self.pause.parked() || self.interrupt.parked();
1908        if !parking && self.state.resource_block.is_some() {
1909            self.stop_for_resource()?;
1910            return Ok(true);
1911        }
1912        // Either handle asking is enough - see `Pause`'s own doc for why
1913        // they are never the same one. `interrupt` is checked second so a
1914        // reason it carries is preferred in the message below over a plain
1915        // shutdown park racing it at the same boundary.
1916        if !self.pause.parked() && !self.interrupt.parked() {
1917            return Ok(false);
1918        }
1919        if self.state.parked {
1920            return Ok(true);
1921        }
1922        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1923        if let Some(block) = &mut self.state.resource_block
1924            && matches!(
1925                block.cause,
1926                crate::run::ResourceBlockCause::Cancelled { .. }
1927            )
1928        {
1929            block.cause = crate::run::ResourceBlockCause::Cancelled {
1930                park: cause.clone(),
1931            };
1932        }
1933        let why = match cause.as_ref().map(ParkCause::label) {
1934            Some(reason) => format!(
1935                "parked after `{}` ({reason}) — resume to carry on from here",
1936                self.state.status.as_str()
1937            ),
1938            None => format!(
1939                "parked after `{}` — resume to carry on from here",
1940                self.state.status.as_str()
1941            ),
1942        };
1943        self.state.event("park", why);
1944        self.state
1945            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1946        self.state.parked = true;
1947        self.state.save()?;
1948        Ok(true)
1949    }
1950
1951    /// Only a new explicit operator approval may retry failed preparation.
1952    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1953        if self.state.prep_failure.is_none() {
1954            return Ok(());
1955        }
1956        let failed_at = self
1957            .state
1958            .events
1959            .iter()
1960            .rev()
1961            .find(|event| event.node == "prep")
1962            .map_or(self.state.updated_at, |event| event.at);
1963        anyhow::ensure!(
1964            approval_at > failed_at,
1965            "preparation retry requires a new explicit release after the failure"
1966        );
1967        self.state.prep_failure = None;
1968        self.state.prep_complete = Some(false);
1969        self.state
1970            .event("prep-retry", "operator approved retry of preparation");
1971        Ok(())
1972    }
1973
1974    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1975    pub fn on_pause(&mut self, pause: Pause) {
1976        self.pause = pause;
1977    }
1978
1979    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1980    /// scheduler asking this one run - and no other - to park so a task
1981    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1982    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1983    /// handle.
1984    pub fn watch_interrupt(&mut self, pause: Pause) {
1985        self.interrupt = pause;
1986    }
1987
1988    /// Abandon this run's own open questions, once `status` has actually
1989    /// settled rather than merely paused.
1990    ///
1991    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1992    /// either back up with the candidates, the review round and the seat
1993    /// sessions already on disk, so a question an implementer asked mid-round
1994    /// may still get a real answer read by a real resume. Only the statuses
1995    /// `resumable` excludes are actually final: the run merged, it reached
1996    /// `Ready` with nothing left to do, it failed outright with no
1997    /// established point to continue from, or every candidate agreed, with
1998    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1999    /// every one of those the seat that asked is gone for good, exactly like
2000    /// the run being deleted under `magi run rm` - so the same cleanup
2001    /// applies, worded for what actually happened instead of "the run was
2002    /// deleted".
2003    ///
2004    /// Best-effort and silent on success: called from every place `status`
2005    /// can land on one of those three, including ones a resumed run revisits,
2006    /// so it must cost nothing when there was nothing open to begin with.
2007    fn settle_questions(&mut self) {
2008        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2009            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2010        }
2011    }
2012
2013    /// The tail of the graph after a trustworthy tally: fold losers, review,
2014    /// gate, merge, and persist.
2015    async fn finish_after_tally(&mut self) -> Result<()> {
2016        // Before the losers are folded: their branches are what the graft
2017        // reads from.
2018        self.graft().await?;
2019        if self.resource_halt()? {
2020            return Ok(());
2021        }
2022        self.fold_losers().await?;
2023        // Before review starts, and again right before the gate: a run's
2024        // review rounds can themselves take long enough for the base to move
2025        // a second time, and the gate is the one node whose "green" gets
2026        // acted on.
2027        self.sync_to_base().await?;
2028        if self.hold_base_sync().await? || self.resource_halt()? {
2029            return Ok(());
2030        }
2031        if self.state.status == RunStatus::AlreadyInBase {
2032            return Ok(());
2033        }
2034        self.review_loop().await?;
2035        if self.resource_halt()? {
2036            return Ok(());
2037        }
2038        self.sync_to_base().await?;
2039        if self.hold_base_sync().await? || self.resource_halt()? {
2040            return Ok(());
2041        }
2042        if self.state.status == RunStatus::AlreadyInBase {
2043            return Ok(());
2044        }
2045        self.gate().await?;
2046        self.merge().await?;
2047        if self.resource_halt()? {
2048            return Ok(());
2049        }
2050        self.state.save()?;
2051        Ok(())
2052    }
2053
2054    // ---------------------------------------------------------------- prep
2055
2056    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2057    /// run with the step and its output in the event; it is never swallowed.
2058    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2059    /// resume starts over instead of finding candidates and missing seats.
2060    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2061        let result = match ensure_setup_config(&mut self.state, repo).await {
2062            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2063            Err(e) => Err(e),
2064        };
2065        if let Err(e) = result {
2066            git::worktree_remove_clean(repo, wt).await.ok();
2067            if in_prep {
2068                self.rollback_prep(repo).await;
2069            }
2070            self.state.status = RunStatus::Blocked;
2071            self.state.event(
2072                "setup",
2073                format!("worktree setup failed in {}: {e:#}", wt.display()),
2074            );
2075            self.state.save()?;
2076            return Err(e);
2077        }
2078        Ok(())
2079    }
2080
2081    /// Undo everything `prep` created, so a resume re-runs it whole.
2082    async fn rollback_prep(&mut self, repo: &Path) {
2083        let root = self.state.worktree_root();
2084        for c in std::mem::take(&mut self.state.candidates) {
2085            if matches!(
2086                git::worktree_remove_clean(repo, &c.worktree).await,
2087                Ok(true)
2088            ) {
2089                git::branch_delete(repo, &c.branch).await.ok();
2090            } else {
2091                self.state.candidates.push(c);
2092            }
2093        }
2094        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2095        for j in 1..=seats {
2096            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2097                .await
2098                .ok();
2099        }
2100        for k in 1..=self.state.config.graph.advisors {
2101            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2102                .await
2103                .ok();
2104        }
2105    }
2106
2107    async fn prep(&mut self) -> Result<()> {
2108        if let Some(reason) = &self.state.prep_failure {
2109            bail!(
2110                "candidate preparation previously failed: {reason}; correct the source before retrying"
2111            );
2112        }
2113        if self
2114            .state
2115            .prep_complete
2116            .unwrap_or(!self.state.candidates.is_empty())
2117        {
2118            return Ok(());
2119        }
2120        let repo = self.state.repo.clone();
2121        // Includes rollback, config, refs, checkout creation and setup. No
2122        // repository mutation may precede the real shared lock acquisition.
2123        let _coordination = self.coordinate(&repo).await?;
2124        let base = self.state.base_commit.clone();
2125        let plan = self.state.sources.plan(&repo, &base).await?;
2126        self.rollback_prep(&repo).await;
2127        if !self.state.candidates.is_empty() {
2128            bail!(
2129                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2130            );
2131        }
2132        self.state.prep_complete = Some(false);
2133        self.state.status = RunStatus::Prep;
2134        self.state.save()?;
2135        let repo = self.state.repo.clone();
2136        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2137        let root = self.state.worktree_root();
2138        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2139
2140        // The hook is the write-time half of the blindness contract; the
2141        // presentation filter in `blind` is the half that cannot be bypassed.
2142        let hooks_dir = self.state.dir().join("hooks");
2143        if self.state.config.blind.commit_msg_hook {
2144            std::fs::create_dir_all(&hooks_dir)
2145                .with_context(|| format!("create {}", hooks_dir.display()))?;
2146            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2147            let path = hooks_dir.join("commit-msg");
2148            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2149            make_executable(&path)?;
2150            // Ref-counted rather than a plain idempotent set: with more than
2151            // one run able to be in flight in the same repository at once
2152            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2153            // true?" check cannot tell "another run of mine still needs
2154            // this" from "nobody does", and the run that happens to finish
2155            // first would disable the hook out from under a sibling still
2156            // relying on it.
2157            git::acquire_worktree_config(&repo).await?;
2158            self.state.enabled_worktree_config = true;
2159        }
2160
2161        // `[worktree] setup` hides its products per worktree, which needs the
2162        // same per-worktree config the hook does. Held until fold, like it.
2163        ensure_setup_config(&mut self.state, &repo).await?;
2164
2165        for (index, (spec, label)) in self
2166            .roles
2167            .implementers
2168            .clone()
2169            .into_iter()
2170            .zip(labels)
2171            .enumerate()
2172        {
2173            let branch = self.state.branch_for(label);
2174            let worktree = root.join(format!("cand-{label}"));
2175            // Persist intent before Git can leave a partially created tree.
2176            // Candidate registration follows setup, so it cannot inventory
2177            // crashes or refusals during creation itself.
2178            self.state.event(
2179                "prep-intent",
2180                serde_json::json!({
2181                    "branch": branch,
2182                    "worktree": worktree,
2183                    "start": start,
2184                    "picks": plan.picks,
2185                })
2186                .to_string(),
2187            );
2188            self.state.save()?;
2189            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2190            if self.state.config.blind.commit_msg_hook {
2191                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2192            }
2193            git::local_exclude(&worktree, "/.magi/").await?;
2194            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2195                // Nothing of this prep may survive: a resume must start over,
2196                // not find half the candidates and skip the rest.
2197                git::worktree_remove_clean(&repo, &worktree).await.ok();
2198                git::branch_delete(&repo, &branch).await.ok();
2199                self.rollback_prep(&repo).await;
2200                self.state.status = RunStatus::Blocked;
2201                self.state
2202                    .event("prep", format!("worktree setup failed: {e:#}"));
2203                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2204                self.state.save()?;
2205                return Err(e);
2206            }
2207            self.state.candidates.push(Candidate {
2208                index,
2209                label,
2210                agent: spec.id.clone(),
2211                branch,
2212                worktree,
2213                summary: String::new(),
2214                stat: String::new(),
2215                files: 0,
2216                commits: 0,
2217                empty: false,
2218                failed: None,
2219                verified_noop: None,
2220                duration_ms: 0,
2221                folded: false,
2222            });
2223            self.state.save()?;
2224            for pick in &plan.picks {
2225                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2226                {
2227                    self.state.status = RunStatus::Blocked;
2228                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2229                    self.state
2230                        .event("prep", format!("cannot apply referenced commit: {e}"));
2231                    self.state.save()?;
2232                    return Err(e);
2233                }
2234            }
2235        }
2236
2237        for j in 1..=self.roles.judges.len() {
2238            let wt = root.join(format!("judge-{j}"));
2239            if !wt.exists() {
2240                git::worktree_add_detached(&repo, &wt, &base).await?;
2241                self.setup_seat_worktree(&repo, &wt, true).await?;
2242            }
2243        }
2244
2245        // Disposable, detached checkouts for the design-deliberation stage's
2246        // advisor seats — the same shape as the judges' above, at the same
2247        // base commit, since advisors also only ever read. Sized off the
2248        // configured count directly rather than a resolved roster: unlike
2249        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2250        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2251        // `prep` has no `ResolvedRoles` field to read a count from here.
2252        if self.state.config.graph.advise {
2253            for k in 1..=self.state.config.graph.advisors {
2254                let wt = root.join(format!("advisor-{k}"));
2255                if !wt.exists() {
2256                    git::worktree_add_detached(&repo, &wt, &base).await?;
2257                    self.setup_seat_worktree(&repo, &wt, true).await?;
2258                }
2259            }
2260        }
2261
2262        // A judge cannot tell it is looking at its own patch — the seats keep
2263        // separate conversations — but a panel that shares agents with the
2264        // field is less independent than it looks, and that is worth saying out
2265        // loud once per run rather than leaving it in the config.
2266        let authors: Vec<&str> = self
2267            .roles
2268            .implementers
2269            .iter()
2270            .map(|a| a.id.as_str())
2271            .collect();
2272        let overlap: Vec<String> = self
2273            .roles
2274            .judges
2275            .iter()
2276            .enumerate()
2277            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2278            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2279            .collect();
2280        if !overlap.is_empty() {
2281            let note = format!(
2282                "{} also authored a candidate; blind, but the panel is less \
2283                 independent than {} distinct agents would be",
2284                overlap.join(", "),
2285                self.roles.judges.len()
2286            );
2287            self.state.event("prep", note);
2288        }
2289
2290        self.state.event(
2291            "prep",
2292            format!(
2293                "{} candidates, {} judges, base {} ({})",
2294                self.state.candidates.len(),
2295                self.roles.judges.len(),
2296                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2297                self.state.base_branch
2298            ),
2299        );
2300        self.state.prep_complete = Some(true);
2301        self.state.status = RunStatus::Implementing;
2302        self.state.save()?;
2303        Ok(())
2304    }
2305
2306    // -------------------------------------------------------------- advise
2307
2308    /// The design-deliberation stage: independent, read-only advisor seats
2309    /// each sketch a design before any implementer touches the repository,
2310    /// and (when at least one produced a usable proposal) a synthesis seat
2311    /// blends them into a brief `implement` carries in every candidate's
2312    /// prompt.
2313    ///
2314    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2315    /// advisors` is the proposal count. Everything here is best-effort and
2316    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2317    /// that cannot reach quota, or a synthesis seat that produced nothing
2318    /// usable all leave `implement` exactly as it was before this stage
2319    /// existed — the task instruction alone — rather than failing the whole
2320    /// competition over an enrichment stage. Every outcome is still recorded
2321    /// as an event, so a run that got nothing from this stage says why.
2322    ///
2323    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2324    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2325    /// resumed run whose stage failed would re-run it, and re-spend the
2326    /// agent calls, on every reentry before `implement`.
2327    ///
2328    /// Also skipped once any candidate shows implementation progress — the
2329    /// exact predicate `implement` itself uses to decide a candidate is no
2330    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2331    /// is not enough: a run created by an older binary that predates this
2332    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2333    /// an already-`Implementing`-or-later run under this build would
2334    /// otherwise walk straight back through `prep` (a no-op once candidates
2335    /// exist) into this node and spawn every advisor seat against worktrees
2336    /// `prep` never recreated — after implementation has already started,
2337    /// which is exactly the invariant this stage exists to guarantee.
2338    async fn advise(&mut self) -> Result<()> {
2339        let implement_untouched = self
2340            .state
2341            .candidates
2342            .iter()
2343            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2344        if !self.state.config.graph.advise || self.state.advise_attempted {
2345            return Ok(());
2346        }
2347        if !implement_untouched {
2348            self.state.event(
2349                "advise",
2350                "skipping the design-deliberation stage: at least one \
2351                 candidate already shows implementation progress, so this \
2352                 run is past the point the stage exists to run before"
2353                    .to_owned(),
2354            );
2355            self.state.advise_attempted = true;
2356            self.state.save()?;
2357            return Ok(());
2358        }
2359        let run_id = self.state.id.clone();
2360        let prompts = self.state.config.prompts.clone();
2361        let instruction = self.state.instruction.clone();
2362        let language = self.state.config.graph.language.clone();
2363        let root = self.state.worktree_root();
2364        let n = self.state.config.graph.advisors;
2365        let where_recorded = self.state.dir().join("run.json");
2366
2367        let seats = match self.state.config.advisors() {
2368            Ok(seats) if !seats.is_empty() => seats,
2369            Ok(_) => {
2370                self.state.event(
2371                    "advise",
2372                    format!(
2373                        "[graph] advisors is 0; skipping the design-deliberation \
2374                         stage and continuing without a synthesis brief (see {})",
2375                        where_recorded.display()
2376                    ),
2377                );
2378                self.state.advise_attempted = true;
2379                self.state.save()?;
2380                return Ok(());
2381            }
2382            Err(e) => {
2383                self.state.event(
2384                    "advise",
2385                    format!(
2386                        "could not resolve advisor seats ({e:#}); continuing \
2387                         without a design-deliberation brief (see {})",
2388                        where_recorded.display()
2389                    ),
2390                );
2391                self.state.advise_attempted = true;
2392                self.state.save()?;
2393                return Ok(());
2394            }
2395        };
2396
2397        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2398        let artifacts = agent::artifacts_dir(&self.state.dir());
2399        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2400
2401        let mut jobs = Vec::new();
2402        for (i, spec) in seats.iter().cloned().enumerate() {
2403            let seat_key = format!("advisor-{}", i + 1);
2404            let seat = self.seat(&seat_key, &spec.id);
2405            jobs.push(SeatJob {
2406                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2407                spec,
2408                seat,
2409                cwd: worktrees[i % worktrees.len()].clone(),
2410                timeout,
2411                allow_write: false,
2412                sessions: false,
2413                artifacts: artifacts.clone(),
2414                stem: seat_key,
2415                handover: None,
2416            });
2417        }
2418
2419        self.state.event(
2420            "advise",
2421            format!(
2422                "{} advisor seat(s) sketching a design in parallel",
2423                jobs.len()
2424            ),
2425        );
2426        let mut quota_losses = Vec::new();
2427        let cache = self.state.config.cache_dir();
2428        let ctx = WaveCtx {
2429            carry_seats: false,
2430            pauses: Some((&self.pause, &self.interrupt)),
2431            run: &run_id,
2432            node: "advise",
2433            prompts: &prompts,
2434            cache: cache.as_deref(),
2435            round: None,
2436        };
2437        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2438        let results = ask_json_wave::<Proposal>(
2439            jobs,
2440            Arc::clone(&self.sem),
2441            self.state.config.graph.retries,
2442            &advisor_roster,
2443            &ctx,
2444            &mut quota_losses,
2445            &mut self.state,
2446            &|p: &Proposal| p.validate(),
2447        )
2448        .await;
2449        if self.state.resource_block.is_some() {
2450            return self.stop_for_resource();
2451        }
2452        self.state.quota.extend(quota_losses);
2453
2454        let mut records = Vec::with_capacity(results.len());
2455        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2456            let agent_id = seat.agent.clone();
2457            self.state.seats.insert(seat.key.clone(), seat);
2458            match res {
2459                Ok((proposal, out)) => {
2460                    self.state
2461                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2462                    records.push(advise::AdvisorRecord::proposed(
2463                        i + 1,
2464                        agent_id,
2465                        proposal,
2466                        out.duration_ms,
2467                    ));
2468                }
2469                Err(e) => {
2470                    self.state.event(
2471                        "advise",
2472                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2473                    );
2474                    records.push(advise::AdvisorRecord::failed(
2475                        i + 1,
2476                        agent_id,
2477                        e.to_string(),
2478                    ));
2479                }
2480            }
2481        }
2482
2483        let mut advice = advise::Advice {
2484            records,
2485            synthesis: None,
2486        };
2487        if advice.proposals().is_empty() {
2488            self.state.event(
2489                "advise",
2490                "no advisor produced a usable proposal; continuing without a \
2491                 synthesis brief"
2492                    .to_owned(),
2493            );
2494        } else {
2495            match self
2496                .synthesize_brief(
2497                    &advice,
2498                    &instruction,
2499                    &language,
2500                    &worktrees[0],
2501                    &artifacts,
2502                    &run_id,
2503                    &prompts,
2504                    cache.as_deref(),
2505                )
2506                .await
2507            {
2508                Ok(Some(text)) => {
2509                    self.state.event(
2510                        "advise",
2511                        "synthesized a design brief for the implementer".to_owned(),
2512                    );
2513                    advice.synthesis = Some(text);
2514                }
2515                Ok(None) => {
2516                    self.state.event(
2517                        "advise",
2518                        "the synthesis seat produced nothing usable; continuing \
2519                         without a design brief"
2520                            .to_owned(),
2521                    );
2522                }
2523                Err(e) => {
2524                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2525                        return Err(e);
2526                    }
2527                    self.state.event(
2528                        "advise",
2529                        format!("could not synthesize a design brief: {e:#}"),
2530                    );
2531                }
2532            }
2533        }
2534        advise::apply_reflection(&mut advice);
2535
2536        self.state.advice = Some(advice);
2537        self.state.advise_attempted = true;
2538        self.state.save()?;
2539        Ok(())
2540    }
2541
2542    /// The synthesis seat: reads every advisor's proposal and blends them
2543    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2544    /// out of [`Runner::advise`] only for readability — it is not called
2545    /// anywhere else.
2546    ///
2547    /// Picked the same way [`crate::talk`]'s standing conversation and
2548    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2549    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2550    /// order (a claude seat, else the first runnable agent in roster order)
2551    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2552    /// in [`crate::config`] for why a dedicated field exists here at all.
2553    #[allow(clippy::too_many_arguments)]
2554    async fn synthesize_brief(
2555        &mut self,
2556        advice: &advise::Advice,
2557        instruction: &str,
2558        language: &str,
2559        cwd: &Path,
2560        artifacts: &Path,
2561        run_id: &str,
2562        prompts: &Prompts,
2563        cache: Option<&Path>,
2564    ) -> Result<Option<String>> {
2565        let chain = agent::pick_chain(
2566            &self.state.config.agents,
2567            self.state.config.roles.synthesizer.as_ref(),
2568            &agent::installed,
2569            "synthesizer",
2570        )?;
2571        let proposals = advice.proposals();
2572        let mut prompt = prompt::with_overlay(
2573            prompt::synthesize_brief(instruction, &proposals, language),
2574            prompts.overlay("advise"),
2575        );
2576        if cache.is_some() {
2577            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2578            // below — see `prompt::build_cache_note`'s doc for why telling a
2579            // read-only seat to build through the shared cache is exactly how
2580            // a sandbox's write refusal gets misread as a defect.
2581            prompt.push('\n');
2582            prompt.push_str(&prompt::build_cache_note("advise", false));
2583        }
2584        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2585        // Each id is tried once, in order; a quota hit, error or unusable
2586        // answer moves to the next. The seat is single-turn (`sessions:
2587        // false`) and the prompt is the whole context, so a fallback agent
2588        // needs nothing carried over.
2589        let mut last = None;
2590        for (n, spec) in chain.iter().enumerate() {
2591            if n > 0 {
2592                self.state
2593                    .event("advise", format!("synthesis falling back to {}", spec.id));
2594            }
2595            let mut seat = self.seat("advise-synthesis", &spec.id);
2596            let wait = self.repository_wait();
2597            let outcome = wait
2598                .scope(agent::invoke(
2599                    spec,
2600                    &mut seat,
2601                    &Invocation {
2602                        coordination: self.state.config.coordination.as_ref(),
2603                        fetched_base: None,
2604                        cwd,
2605                        prompt: &prompt,
2606                        timeout,
2607                        allow_write: false,
2608                        sessions: false,
2609                        artifacts,
2610                        stem: &if n == 0 {
2611                            "advise-synthesis".to_owned()
2612                        } else {
2613                            format!("advise-synthesis-{}", spec.id)
2614                        },
2615                        run: run_id,
2616                        node: "advise",
2617                        cache_dir: None,
2618                        attachments: &[],
2619                        writable: &[],
2620                    },
2621                ))
2622                .await;
2623            if outcome.is_ok() {
2624                self.state.seats.insert(seat.key.clone(), seat);
2625            }
2626            let advance = agent::chain_advances(&outcome);
2627            last = Some(outcome);
2628            if !advance {
2629                break;
2630            }
2631        }
2632        // Exhausted: the last attempt's result is what a single failed seat
2633        // would have produced.
2634        let out = last.expect("a chain holds at least one agent")?;
2635        if !out.usable() {
2636            return Ok(None);
2637        }
2638        let text =
2639            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2640        Ok((!text.trim().is_empty()).then_some(text))
2641    }
2642
2643    // ----------------------------------------------------------- implement
2644
2645    async fn implement(&mut self) -> Result<()> {
2646        self.persist_candidates().await?;
2647        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2648        // agent files with `magi task add` name the run that paid for it. The
2649        // prompt overlay is cloned alongside it because the waves borrow it
2650        // while `self` is mutably borrowed by the node's own bookkeeping.
2651        let run_id = self.state.id.clone();
2652        let prompts = self.state.config.prompts.clone();
2653        let todo: Vec<usize> = self
2654            .state
2655            .candidates
2656            .iter()
2657            .enumerate()
2658            .filter(|(_, c)| {
2659                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2660                    && c.failed.is_none()
2661                    && !c.empty
2662            })
2663            .map(|(i, _)| i)
2664            .collect();
2665        if todo.is_empty() {
2666            return self.after_implement();
2667        }
2668        self.state.status = RunStatus::Implementing;
2669
2670        let language = self.state.config.graph.language.clone();
2671        let github_language = self.state.config.graph.github_language.clone();
2672        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2673        let sessions = self.state.config.graph.sessions;
2674        let artifacts = agent::artifacts_dir(&self.state.dir());
2675        // The design-deliberation stage's blended brief, when `advise` found
2676        // one — carried into every implementer's prompt the same way
2677        // regardless of which candidate it is.
2678        let brief = self
2679            .state
2680            .advice
2681            .as_ref()
2682            .and_then(|a| a.synthesis.as_deref())
2683            .map(str::to_owned);
2684        let attachments = self.state.attachments.clone();
2685
2686        let mut jobs = Vec::new();
2687        for &i in &todo {
2688            let (index, label, worktree) = {
2689                let c = &self.state.candidates[i];
2690                (c.index, c.label, c.worktree.clone())
2691            };
2692            let spec = self.roles.implementers[index].clone();
2693            let seat_key = format!("impl-{label}");
2694            let seat = self.seat(&seat_key, &spec.id);
2695            let instruction = seeded_instruction(&self.state);
2696            jobs.push(SeatJob {
2697                spec,
2698                seat,
2699                prompt: prompt::implement_in(
2700                    &instruction,
2701                    &worktree.to_string_lossy(),
2702                    &language,
2703                    &github_language,
2704                    brief.as_deref(),
2705                    &attachments,
2706                ),
2707                cwd: worktree,
2708                timeout,
2709                allow_write: true,
2710                sessions,
2711                artifacts: artifacts.clone(),
2712                stem: format!("impl-{label}"),
2713                handover: None,
2714            });
2715        }
2716
2717        self.state.event(
2718            "implement",
2719            format!("{} candidates in parallel", jobs.len()),
2720        );
2721        // Kept so a seat whose CLI hung up can be asked again from the same
2722        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2723        // can update a seat's own entry once a fallback agent takes it over —
2724        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2725        // whichever agent actually answered, not the one that quota'd out.
2726        let mut sent = jobs.clone();
2727        let cache = self.state.config.cache_dir();
2728        let ctx = WaveCtx {
2729            carry_seats: false,
2730            pauses: Some((&self.pause, &self.interrupt)),
2731            run: &run_id,
2732            node: "implement",
2733            prompts: &prompts,
2734            cache: cache.as_deref(),
2735            round: None,
2736        };
2737        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2738        if !wave_blocked(&results) {
2739            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2740                .await;
2741            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2742                .await?;
2743            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2744                .await;
2745        }
2746        // Do not finalize failed candidates when the entire wave needs machine
2747        // recovery: leaving them pending resumes this node with its work intact.
2748        if !results.is_empty()
2749            && results
2750                .iter()
2751                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
2752        {
2753            let losses: Vec<_> = results
2754                .iter()
2755                .map(|(_, seat, out)| {
2756                    let AgentOutcome::Quota(o) = out else {
2757                        unreachable!()
2758                    };
2759                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
2760                })
2761                .collect();
2762            self.state.quota.extend(losses.clone());
2763            for (_, seat, _) in results {
2764                self.state.seats.insert(seat.key.clone(), seat);
2765            }
2766            for &i in &todo {
2767                let label = self.state.candidates[i].label;
2768                self.state.interrupted_implementations.insert(label);
2769                self.state
2770                    .candidate_preservation
2771                    .insert(label, "machine outage awaiting persistence".into());
2772            }
2773            // Preserve partial edits too, while keeping every interrupted seat
2774            // pending even when its tree already contains an inherited patch.
2775            block_machine_outage(&mut self.state, "implement", &losses);
2776            self.state.save()?;
2777            self.persist_candidates().await?;
2778            for &i in &todo {
2779                self.state
2780                    .interrupted_implementations
2781                    .insert(self.state.candidates[i].label);
2782            }
2783            return self.stop_for_resource();
2784        }
2785        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2786
2787        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2788            && results
2789                .iter()
2790                .any(|(_, _, out)| FailClass::of(out).is_some());
2791        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2792            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2793                // Executed turns interrupted by disk pressure already carry
2794                // a preservation barrier from wave. Admission-only waits do
2795                // not. Keep the interrupted seat's session for recovery too.
2796                if self
2797                    .state
2798                    .candidate_preservation
2799                    .contains_key(&self.state.candidates[i].label)
2800                {
2801                    self.state.seats.insert(seat.key.clone(), seat);
2802                }
2803                continue;
2804            }
2805            let seat_key = seat.key.clone();
2806            // A quota fallback (`resume_seat_handovers`) may have handed this
2807            // seat to a different agent than the one `prep` recorded on the
2808            // candidate; the stats tables and any later fixer-defaults-to-
2809            // winner's-author lookup must credit whoever actually answered —
2810            // unless every fallback also quota'd out, in which case nobody
2811            // actually answered and crediting the last agent tried would
2812            // erase every earlier agent's own quota loss from the stats
2813            // tables instead of just this one seat's.
2814            let agent = seat.agent.clone();
2815            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2816            self.state.seats.insert(seat.key.clone(), seat);
2817            let label = self.state.candidates[i].label;
2818            self.state.interrupted_implementations.remove(&label);
2819
2820            let (summary, duration, failed, verified_claim) = match out {
2821                AgentOutcome::Ok(o) => {
2822                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2823                    let failed = (!o.usable()).then(|| {
2824                        if o.timed_out {
2825                            "agent timed out".to_owned()
2826                        } else {
2827                            format!(
2828                                "agent exited with {}",
2829                                crate::notices::exit_status(o.exit_code)
2830                            )
2831                        }
2832                    });
2833                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2834                    (text, o.duration_ms, failed, verified_claim)
2835                }
2836                // Left un-resumed by `resume_undelivered` (a dirty tree
2837                // already rescues the work, or there was no session left to
2838                // resume into) — reported like the ordinary failure it is,
2839                // never as if `o.text` (the CLI's raw error JSON) were an
2840                // answer.
2841                AgentOutcome::Dropped(o) => {
2842                    let why = o
2843                        .dropped
2844                        .as_ref()
2845                        .map(|d| d.why.as_str())
2846                        .unwrap_or("the CLI ended the stream without delivering its answer");
2847                    (
2848                        String::new(),
2849                        o.duration_ms,
2850                        Some(format!("the CLI dropped the stream ({why})")),
2851                        None,
2852                    )
2853                }
2854                AgentOutcome::Quota(o) => {
2855                    self.state.quota.push(QuotaLoss::from_report(
2856                        seat_key,
2857                        "implement",
2858                        o.quota.as_ref(),
2859                    ));
2860                    (
2861                        String::new(),
2862                        o.duration_ms,
2863                        Some("rate limited (quota); produced no change".to_owned()),
2864                        None,
2865                    )
2866                }
2867                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2868                    (String::new(), 0, Some(e), None)
2869                }
2870            };
2871
2872            // Save every completed turn before any fallible Git operation. A failed
2873            // rescue must not erase the reply or prevent collection of other seats.
2874            let c = &mut self.state.candidates[i];
2875            if !exhausted_the_fallback_chain {
2876                c.agent = agent;
2877            }
2878            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2879            c.duration_ms = duration;
2880            c.failed = failed;
2881            c.verified_noop = verified_claim;
2882            self.state
2883                .candidate_preservation
2884                .insert(label, "awaiting persistence".to_owned());
2885        }
2886        self.state.save()?;
2887        self.persist_candidates().await?;
2888
2889        if resource_blocked {
2890            return self.stop_for_resource();
2891        }
2892        if selected_failed {
2893            self.state.status = RunStatus::Blocked;
2894            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2895            self.state.save()?;
2896            bail!(
2897                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2898            );
2899        }
2900        self.after_implement()
2901    }
2902
2903    /// Retry only persistence after a completed implementation turn.
2904    async fn persist_candidates(&mut self) -> Result<()> {
2905        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2906        for label in labels {
2907            let i = self
2908                .state
2909                .candidates
2910                .iter()
2911                .position(|c| c.label == label)
2912                .context("pending candidate is missing")?;
2913            let c = self.state.candidates[i].clone();
2914            let interrupted = self.state.interrupted_implementations.contains(&label);
2915            let result = async {
2916                let wait = crate::repo_lock::WaitPolicy::new(
2917                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2918                    || false,
2919                );
2920                let _coordination =
2921                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2922                // Best effort evidence requires no index writes. A successful snapshot
2923                // never authorizes removal of a tree whose rescue failed.
2924                if let Err(e) = git::snapshot_uncommitted(
2925                    &c.worktree,
2926                    &self
2927                        .state
2928                        .dir()
2929                        .join("artifacts")
2930                        .join(format!("candidate-{label}-recovery")),
2931                )
2932                .await
2933                {
2934                    self.state.event(
2935                        "preserve",
2936                        format!("candidate {label} snapshot incomplete: {e:#}"),
2937                    );
2938                }
2939                let rescued = git::rescue_commit(
2940                    &c.worktree,
2941                    &format!("magi: candidate {label} (uncommitted work)"),
2942                )
2943                .await?;
2944                self.state.note_withheld("implement", &rescued.withheld);
2945                let base = &self.state.base_commit;
2946                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2947                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2948                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2949                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2950                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2951                if c.empty && commits > 0 && !patch.trim().is_empty()
2952                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2953                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2954                        || self.state.tally.is_some())
2955                {
2956                    // Keep the old classification and pending marker so every
2957                    // retry stops too. Retain decisions as recovery evidence;
2958                    // only a supported recovery operation may replace them.
2959                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2960                }
2961                let c = &mut self.state.candidates[i];
2962                c.commits = commits;
2963                c.stat = stat;
2964                c.files = files;
2965                let no_changes = commits == 0 || patch.trim().is_empty();
2966                c.empty = no_changes && !interrupted;
2967                if interrupted && no_changes {
2968                    c.failed = None;
2969                    c.verified_noop = None;
2970                }
2971                if !c.empty {
2972                    c.failed = None;
2973                    c.verified_noop = None;
2974                }
2975                self.state.event(
2976                    "preserve",
2977                    format!("candidate {label}: {files} files, {commits} commits"),
2978                );
2979                Ok::<_, anyhow::Error>(())
2980            }
2981            .await;
2982            match result {
2983                Ok(()) => {
2984                    self.state.candidate_preservation.remove(&label);
2985                    if !interrupted || self.state.candidates[i].files > 0 {
2986                        self.state.interrupted_implementations.remove(&label);
2987                    }
2988                }
2989                Err(e) => {
2990                    let reason = format!("candidate {label} preservation failed: {e:#}");
2991                    self.state.event("preserve", &reason);
2992                    self.state.candidate_preservation.insert(label, reason);
2993                }
2994            }
2995            if self.state.candidate_preservation.is_empty()
2996                && self.state.status == RunStatus::Blocked
2997                && self.state.candidates.iter().all(|c| c.failed.is_none())
2998            {
2999                // Persist the recovered phase with the cleared pending marker;
3000                // a crash between saves must not turn this into a selected-seat
3001                // failure hold on the next resume.
3002                self.state.status = RunStatus::Implementing;
3003                self.state.event(
3004                    "preserve",
3005                    "candidate persistence recovered; implementation can continue",
3006                );
3007            }
3008            self.state.save()?;
3009        }
3010        if !self.state.candidate_preservation.is_empty() {
3011            self.state.status = RunStatus::Blocked;
3012            self.state.save()?;
3013            bail!(
3014                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3015            );
3016        }
3017        Ok(())
3018    }
3019
3020    /// Ask again, once, for work a CLI did and then failed to hand over.
3021    ///
3022    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3023    /// status with an empty response and a usage report showing output tokens,
3024    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3025    /// seven minutes and 14,267 output tokens that arrived as an empty
3026    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3027    ///
3028    /// Two conditions, and both matter:
3029    ///
3030    /// - **Only when the tree is untouched.** Often the agent has already
3031    ///   written its files and only the closing message was lost; the rescue
3032    ///   commit below picks that up and there is nothing to ask for. Re-asking
3033    ///   then would pay for a second implementation of work already on disk.
3034    /// - **Once.** A CLI that drops one stream can drop the next, and this
3035    ///   node is the most expensive in the graph.
3036    ///
3037    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3038    /// dropped reply still carried its `conversation_id`, so the seat is asked
3039    /// to finish what it was doing rather than sent the whole task again. It
3040    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3041    /// node's - for the same reason a re-ranked judge does: restating finished
3042    /// work is not the work.
3043    ///
3044    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3045    /// same way until it resets, while an abandoned conversation is still
3046    /// there to be picked up.
3047    async fn resume_undelivered(
3048        &mut self,
3049        results: &mut [(usize, SeatState, AgentOutcome)],
3050        sent: &[SeatJob],
3051        prompts: &Prompts,
3052        run_id: &str,
3053    ) {
3054        for (wi, seat, out) in results.iter_mut() {
3055            let Some(dropped) = (match &*out {
3056                AgentOutcome::Dropped(o) => o.dropped.clone(),
3057                _ => None,
3058            }) else {
3059                continue;
3060            };
3061            let Some(job) = sent.get(*wi) else { continue };
3062            // Already on disk? Then only the closing message was lost.
3063            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3064                self.state.event(
3065                    "implement",
3066                    format!(
3067                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3068                         work is in the tree",
3069                        seat.key, dropped.output_tokens, dropped.why
3070                    ),
3071                );
3072                continue;
3073            }
3074            // The re-ask only makes sense as a resume: `resume_after_drop`
3075            // says nothing about the task, trusting the seat to still hold it.
3076            // Without a session to resume — sessions disabled, or this CLI's
3077            // drop shape happened not to carry a session id — that prompt
3078            // would open a brand-new conversation with no context at all,
3079            // which is worse than leaving this as the ordinary failure it
3080            // already is.
3081            if !has_context(&job.spec, seat, job.sessions) {
3082                self.state.event(
3083                    "implement",
3084                    format!(
3085                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3086                         is no session left to resume",
3087                        seat.key, dropped.output_tokens, dropped.why
3088                    ),
3089                );
3090                continue;
3091            }
3092            self.state.event(
3093                "implement",
3094                format!(
3095                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3096                     conversation",
3097                    seat.key, dropped.output_tokens, dropped.why
3098                ),
3099            );
3100            let mut retry = job.clone();
3101            retry.seat = seat.clone();
3102            retry.prompt = prompt::resume_after_drop(&dropped.why);
3103            retry.timeout = retry_budget(job.timeout, true);
3104            retry.stem = format!("{}-resume", job.stem);
3105            let cache = self.state.config.cache_dir();
3106            let ctx = WaveCtx {
3107                carry_seats: false,
3108                pauses: Some((&self.pause, &self.interrupt)),
3109                run: run_id,
3110                node: "implement",
3111                prompts,
3112                cache: cache.as_deref(),
3113                round: None,
3114            };
3115            let (resumed_seat, resumed) =
3116                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3117            *seat = resumed_seat;
3118            *out = resumed;
3119        }
3120    }
3121
3122    /// Fall an implement seat through to the next untried agent in the
3123    /// implementer roster when it lost to quota — or, since the handover was
3124    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3125    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3126    /// quota path itself being unchanged — instead of leaving the
3127    /// seat's loss final the moment one agent's account runs dry.
3128    ///
3129    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3130    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3131    /// truncates to the single slot rotation picked, so a solo task whose one
3132    /// implementer hits quota mid-run used to have nothing else to try. This
3133    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3134    /// unrotated roster — which is the only place the *other* candidates in
3135    /// the machine's roster still exist once `implementers` has been cut down
3136    /// to size.
3137    ///
3138    /// Walks forward from just past the seat's own original position in the
3139    /// roster, never wrapping back to the front: a later candidate slot (say
3140    /// `beta`, the roster's second entry) must fall through to the *next*
3141    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3142    /// almost certainly a different candidate's own agent already — and once
3143    /// the roster's tail is exhausted there is nothing left to fall through
3144    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3145    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3146    /// let this retry that id forever. The loop keeps falling through until
3147    /// an attempt lands something other than `Quota` or the roster's tail
3148    /// runs out of untried ids, at which point the seat is left exactly as
3149    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3150    /// `QuotaLoss` recorded, the candidate failed/empty.
3151    ///
3152    /// `sent` is taken mutably and updated with the fallback agent's spec:
3153    /// `resume_unconfirmed_commands`, which runs after this and also reads
3154    /// `sent`, must see whichever agent actually ended up answering the seat
3155    /// — reading the stale, original spec there would check session
3156    /// eligibility against the wrong CLI and could hand a fallback agent's
3157    /// session id to the agent that just lost the seat to quota.
3158    ///
3159    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3160    /// — `self.seat` only reuses state when the agent id is unchanged, so
3161    /// handing it a different id already gets this for free. Reusing the old
3162    /// seat would resume a different CLI's session as if it were a
3163    /// continuation of this one.
3164    ///
3165    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3166    /// a quota loss cuts an agent off mid-turn, so anything already in the
3167    /// tree is unfinished work, not a completed candidate a re-ask would pay
3168    /// for twice. A dirty tree is rescued into a commit first (the same
3169    /// neutral-identity rescue `implement`'s own outcome loop gives every
3170    /// candidate) so the next agent starts clean.
3171    ///
3172    /// The new agent gets the implementer's full prompt and full
3173    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3174    /// it has no session and no context, and is implementing the task from
3175    /// nothing, unlike a resumed drop which is only restating work already
3176    /// done.
3177    ///
3178    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3179    /// `implement` event, never into `self.state.quota` — that is what
3180    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3181    /// go unspent, and a seat that ultimately recovered on its second or
3182    /// third agent is not the stalled panel that check exists to catch. Only
3183    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3184    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3185    /// outcome loop already has — this helper never pushes to it itself.
3186    async fn resume_seat_handovers(
3187        &mut self,
3188        results: &mut [(usize, SeatState, AgentOutcome)],
3189        sent: &mut [SeatJob],
3190        prompts: &Prompts,
3191        run_id: &str,
3192    ) -> Result<()> {
3193        let instruction = seeded_instruction(&self.state);
3194        let language = self.state.config.graph.language.clone();
3195        let github_language = self.state.config.graph.github_language.clone();
3196        let brief = self
3197            .state
3198            .advice
3199            .as_ref()
3200            .and_then(|a| a.synthesis.as_deref())
3201            .map(str::to_owned);
3202        let attachments = self.state.attachments.clone();
3203        for (wi, seat, out) in results.iter_mut() {
3204            // Who holds the other candidate seats of this wave right now
3205            // (earlier handovers already written back to `sent`).
3206            let others: BTreeSet<String> = sent
3207                .iter()
3208                .enumerate()
3209                .filter(|(j, _)| j != wi)
3210                .map(|(_, j)| j.spec.id.clone())
3211                .collect();
3212            let Some(job) = sent.get_mut(*wi) else {
3213                continue;
3214            };
3215            // Where the seat's own original agent sits in the roster — the
3216            // fallback walk starts just past here, never at the front, so a
3217            // later candidate slot's quota loss does not fall back onto an
3218            // earlier slot's own agent.
3219            let start = self
3220                .roles
3221                .implementer_roster
3222                .iter()
3223                .position(|s| s.id == job.spec.id)
3224                .unwrap_or(0);
3225            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3226            let mut fallback_attempt = 0usize;
3227            let mut prev: Option<FailClass> = None;
3228            while let Some(cur) = FailClass::of(&*out) {
3229                if !should_hand_over(prev.as_ref(), &cur) {
3230                    break;
3231                }
3232                let Some(next) =
3233                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3234                        .cloned()
3235                else {
3236                    break;
3237                };
3238                tried.insert(next.id.clone());
3239                fallback_attempt += 1;
3240
3241                let coordination = self.coordinate(&job.cwd).await?;
3242                if let Ok(r) = git::rescue_commit(
3243                    &job.cwd,
3244                    &format!(
3245                        "magi: candidate {} (uncommitted work before {} fallback)",
3246                        seat.key,
3247                        if cur == FailClass::Quota {
3248                            "quota"
3249                        } else {
3250                            "handover"
3251                        }
3252                    ),
3253                )
3254                .await
3255                {
3256                    self.state.note_withheld("implement", &r.withheld);
3257                }
3258                drop(coordination);
3259
3260                record_handover(
3261                    &mut self.state,
3262                    "implement",
3263                    &seat.key,
3264                    &seat.agent,
3265                    &next.id,
3266                    &cur,
3267                    &fail_reason(&*out),
3268                );
3269                prev = Some(cur.clone());
3270
3271                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3272                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3273                // Kept in sync on `sent` itself, not just the local retry: a
3274                // later helper (`resume_unconfirmed_commands`) reads `sent`
3275                // after this one returns and must see whichever agent is now
3276                // occupying the seat, not the one that just quota'd out —
3277                // otherwise it would judge session/continuation eligibility
3278                // by the wrong CLI and could resend a fallback's session id
3279                // to the agent that lost it the seat in the first place.
3280                job.spec = next.clone();
3281                let mut retry = job.clone();
3282                retry.seat = new_seat;
3283                retry.prompt = prompt::implement_in(
3284                    &instruction,
3285                    &job.cwd.to_string_lossy(),
3286                    &language,
3287                    &github_language,
3288                    brief.as_deref(),
3289                    &attachments,
3290                );
3291                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3292                let cache = self.state.config.cache_dir();
3293                let ctx = WaveCtx {
3294                    carry_seats: false,
3295                    pauses: Some((&self.pause, &self.interrupt)),
3296                    run: run_id,
3297                    node: "implement",
3298                    prompts,
3299                    cache: cache.as_deref(),
3300                    round: None,
3301                };
3302                let (fallback_seat, fallback_out) = run_one(
3303                    retry,
3304                    Arc::clone(&self.sem),
3305                    &ctx,
3306                    &mut self.state,
3307                    fallback_attempt,
3308                )
3309                .await;
3310                *seat = fallback_seat;
3311                *out = fallback_out;
3312            }
3313        }
3314        Ok(())
3315    }
3316
3317    /// Ask an implement seat's own CLI to confirm what it started, once, when
3318    /// its reply reported a command whose completion status it never
3319    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3320    /// that does and does not mean.
3321    ///
3322    /// The completion contract this task asks for, extended to `implement`
3323    /// with the same signal `continue_fix_report` reads for the fixer,
3324    /// rather than a keyword search over the reply or a hard requirement on
3325    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3326    /// a candidate's CLI turn ended cleanly while a test run it had started
3327    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3328    /// named in it at all is untouched by this: `commands` is empty, so
3329    /// there is nothing to be unconfirmed.
3330    ///
3331    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3332    /// this is not about recovering edits that might already be on disk, it
3333    /// is about a result the seat itself never vouched for, which resuming
3334    /// asks for regardless of what the tree already holds. Bounded to one
3335    /// attempt for the same reason `resume_undelivered` is — this is the
3336    /// most expensive node in the graph — and a seat that still cannot
3337    /// confirm on that attempt is left as whatever its (possibly still
3338    /// unconfirmed) reply says; this does not invent a new "failed" reason
3339    /// for a candidate that otherwise produced a real, committed change.
3340    async fn resume_unconfirmed_commands(
3341        &mut self,
3342        results: &mut [(usize, SeatState, AgentOutcome)],
3343        sent: &[SeatJob],
3344        prompts: &Prompts,
3345        run_id: &str,
3346    ) {
3347        for (wi, seat, out) in results.iter_mut() {
3348            let AgentOutcome::Ok(o) = &*out else {
3349                continue;
3350            };
3351            if !has_unconfirmed_command(&o.commands) {
3352                continue;
3353            }
3354            let Some(job) = sent.get(*wi) else { continue };
3355            if !has_context(&job.spec, seat, job.sessions) {
3356                self.state.event(
3357                    "implement",
3358                    format!(
3359                        "{}: the reply named a command whose own CLI never confirmed the exit \
3360                         status of, but there is no session left to resume",
3361                        seat.key
3362                    ),
3363                );
3364                continue;
3365            }
3366            self.state.event(
3367                "implement",
3368                format!(
3369                    "{}: the reply named a command whose own CLI never confirmed the exit \
3370                     status of; resuming the conversation",
3371                    seat.key
3372                ),
3373            );
3374            let mut retry = job.clone();
3375            retry.seat = seat.clone();
3376            retry.prompt = prompt::resume_incomplete(
3377                "a command in your last reply had no confirmed exit status",
3378            );
3379            retry.timeout = retry_budget(job.timeout, true);
3380            retry.stem = format!("{}-confirm", job.stem);
3381            let cache = self.state.config.cache_dir();
3382            let ctx = WaveCtx {
3383                carry_seats: false,
3384                pauses: Some((&self.pause, &self.interrupt)),
3385                run: run_id,
3386                node: "implement",
3387                prompts,
3388                cache: cache.as_deref(),
3389                round: None,
3390            };
3391            let (resumed_seat, resumed) =
3392                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3393            *seat = resumed_seat;
3394            *out = resumed;
3395        }
3396    }
3397
3398    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3399    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3400    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3401    /// that motivated this.
3402    ///
3403    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3404    /// own nudge loop already covers for judge/review/vote seats, and not a
3405    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3406    /// implement seats: here the CLI turn genuinely finished while the node's
3407    /// own work — the fixer's account of what it did — had not. Gated purely
3408    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3409    /// reply, never on any wording in it, so a fixer whose valid, first-try
3410    /// `FixReport` happens to mention having waited on a background test is
3411    /// never resumed — the `Ok(report)` branch at the call site returns
3412    /// before this is ever invoked.
3413    ///
3414    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3415    /// attempt ([`retry_budget`]), nothing attempted once the session is
3416    /// gone, and a quota hit ends the loop immediately rather than retrying a
3417    /// rate limit that fails the same way again.
3418    async fn continue_fix_report(
3419        &mut self,
3420        mut seat: SeatState,
3421        parse_err: String,
3422        job: &SeatJob,
3423        prompts: &Prompts,
3424        run_id: &str,
3425        round: usize,
3426    ) -> (
3427        SeatState,
3428        Option<FixReport>,
3429        Option<String>,
3430        ContinuationRecord,
3431    ) {
3432        let mut last_err = parse_err;
3433        let mut cumulative_wait_ms = 0u64;
3434        let mut attempts = 0usize;
3435        loop {
3436            if !has_context(&job.spec, &seat, job.sessions) {
3437                self.state.event(
3438                    "fix",
3439                    format!(
3440                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3441                         session left to resume into"
3442                    ),
3443                );
3444                let outcome = if attempts == 0 {
3445                    ContinuationOutcome::NoSession
3446                } else {
3447                    ContinuationOutcome::Exhausted
3448                };
3449                return (
3450                    seat,
3451                    None,
3452                    Some(format!("unparsable fix report: {last_err}")),
3453                    ContinuationRecord {
3454                        attempts,
3455                        cumulative_wait_ms,
3456                        outcome,
3457                    },
3458                );
3459            }
3460            if attempts >= MAX_FIX_CONTINUATIONS {
3461                self.state.event(
3462                    "fix",
3463                    format!(
3464                        "round {round}: fixer's reply still had no adoption report after \
3465                         {attempts} continuation(s) ({last_err}); giving up"
3466                    ),
3467                );
3468                return (
3469                    seat,
3470                    None,
3471                    Some(format!(
3472                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3473                    )),
3474                    ContinuationRecord {
3475                        attempts,
3476                        cumulative_wait_ms,
3477                        outcome: ContinuationOutcome::Exhausted,
3478                    },
3479                );
3480            }
3481            attempts += 1;
3482            self.state.event(
3483                "fix",
3484                format!(
3485                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3486                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3487                ),
3488            );
3489            let mut retry = job.clone();
3490            retry.seat = seat.clone();
3491            retry.prompt = prompt::resume_incomplete(&last_err);
3492            retry.timeout = retry_budget(job.timeout, true);
3493            retry.stem = format!("{}-continue{attempts}", job.stem);
3494            let cache = self.state.config.cache_dir();
3495            let ctx = WaveCtx {
3496                carry_seats: false,
3497                pauses: Some((&self.pause, &self.interrupt)),
3498                run: run_id,
3499                node: "fix",
3500                prompts,
3501                cache: cache.as_deref(),
3502                round: Some(round),
3503            };
3504            let (resumed_seat, resumed_out) = run_one(
3505                retry,
3506                Arc::clone(&self.sem),
3507                &ctx,
3508                &mut self.state,
3509                attempts,
3510            )
3511            .await;
3512            seat = resumed_seat;
3513            match resumed_out {
3514                AgentOutcome::Ok(o) => {
3515                    cumulative_wait_ms += o.duration_ms;
3516                    match verdict::extract_json::<FixReport>(&o.text) {
3517                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3518                            self.state.event(
3519                                "fix",
3520                                format!(
3521                                    "round {round}: fixer's adoption report recovered after \
3522                                     {attempts} continuation(s)"
3523                                ),
3524                            );
3525                            return (
3526                                seat,
3527                                Some(report),
3528                                None,
3529                                ContinuationRecord {
3530                                    attempts,
3531                                    cumulative_wait_ms,
3532                                    outcome: ContinuationOutcome::Resumed,
3533                                },
3534                            );
3535                        }
3536                        // The report parsed, but this same reply's own
3537                        // CommandEvidence — the identical record `state.jobs`
3538                        // renders — names a command whose CLI never
3539                        // confirmed an exit status. Read together, that is
3540                        // not a resolved answer: keep nudging rather than
3541                        // accept a report standing next to a command the
3542                        // seat's own CLI cannot vouch for.
3543                        Ok(_) => {
3544                            last_err = "the reply parsed, but it reported a command whose own CLI \
3545                                 never confirmed an exit status"
3546                                .to_owned();
3547                        }
3548                        Err(e) => last_err = e.to_string(),
3549                    }
3550                }
3551                AgentOutcome::Quota(o) => {
3552                    cumulative_wait_ms += o.duration_ms;
3553                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3554                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3555                    self.state.quota.push(loss);
3556                    self.state.event(
3557                        "fix",
3558                        format!(
3559                            "round {round}: continuation rate limited (quota); not retrying now"
3560                        ),
3561                    );
3562                    return (
3563                        seat,
3564                        None,
3565                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3566                        ContinuationRecord {
3567                            attempts,
3568                            cumulative_wait_ms,
3569                            outcome: ContinuationOutcome::QuotaLost,
3570                        },
3571                    );
3572                }
3573                AgentOutcome::Dropped(o) => {
3574                    cumulative_wait_ms += o.duration_ms;
3575                    let why = o
3576                        .dropped
3577                        .as_ref()
3578                        .map(|d| d.why.as_str())
3579                        .unwrap_or("the CLI ended the stream without delivering its answer");
3580                    last_err = format!("the CLI dropped the stream ({why})");
3581                }
3582                AgentOutcome::Failed(e) => last_err = e,
3583                // The continuation was never started; the report stays lost
3584                // and the caller records that, without any handover.
3585                AgentOutcome::ResourceBlocked(e) => {
3586                    return (
3587                        seat,
3588                        None,
3589                        Some(format!("the continuation was not started: {e}")),
3590                        ContinuationRecord {
3591                            attempts: attempts.saturating_sub(1),
3592                            cumulative_wait_ms,
3593                            outcome: ContinuationOutcome::NoSession,
3594                        },
3595                    );
3596                }
3597            }
3598        }
3599    }
3600
3601    fn after_implement(&mut self) -> Result<()> {
3602        if !self.state.candidate_preservation.is_empty() {
3603            bail!("candidate preservation incomplete; judging is unsafe");
3604        }
3605        // Scan every candidate patch once the set is complete.
3606        if self.state.leaks.is_empty() {
3607            let cfg = self.state.config.blind.clone();
3608            let mut leaks = Vec::new();
3609            for c in &self.state.candidates {
3610                let Some(patch) =
3611                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3612                else {
3613                    continue;
3614                };
3615                leaks.extend(blind::scan(
3616                    &format!("candidate {} patch", c.label),
3617                    &patch,
3618                    &cfg.vendor_tokens,
3619                ));
3620            }
3621            if !leaks.is_empty() {
3622                let summary = leaks
3623                    .iter()
3624                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3625                    .collect::<Vec<_>>()
3626                    .join(", ");
3627                match cfg.on_leak {
3628                    LeakPolicy::Fail => {
3629                        self.state.status = RunStatus::Failed;
3630                        self.state
3631                            .event("blind", format!("vendor text in a patch: {summary}"));
3632                        self.state.leaks = leaks;
3633                        self.state.save()?;
3634                        self.settle_questions();
3635                        bail!(
3636                            "blind.on_leak = \"fail\" and vendor text reached a \
3637                             judged patch: {summary}"
3638                        );
3639                    }
3640                    LeakPolicy::Redact => self.state.event(
3641                        "blind",
3642                        format!("redacting vendor text for judging: {summary}"),
3643                    ),
3644                    LeakPolicy::Warn => self.state.event(
3645                        "blind",
3646                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3647                    ),
3648                }
3649                self.state.leaks = leaks;
3650            }
3651        }
3652
3653        if self.state.viable().is_empty() {
3654            if self.state.all_candidates_verified_noop() {
3655                // Every candidate agreed, with evidence the adoption guard
3656                // accepted, that nothing belongs in this worktree. That is
3657                // not the same fact as a candidate that simply failed to
3658                // write anything, and settling it as an ordinary `Failed`
3659                // (see `SCHEMA`'s doc for schema 10) is what let two of
3660                // task 391f's attempts burn a retry each re-discovering the
3661                // same already-landed fix. Terminal either way, so `judge`
3662                // must never run over an empty candidate set — unlike the
3663                // `Failed` branch below this returns `Ok`, not an error:
3664                // nothing here failed.
3665                self.state.status = RunStatus::VerifiedNoop;
3666                self.state.save()?;
3667                self.settle_questions();
3668                return Ok(());
3669            }
3670            self.state.status = RunStatus::Failed;
3671            self.state.save()?;
3672            self.settle_questions();
3673            bail!("no candidate produced a change; nothing to judge");
3674        }
3675        self.state.status = RunStatus::Judging;
3676        self.state.save()?;
3677        Ok(())
3678    }
3679
3680    // --------------------------------------------------------------- judge
3681
3682    async fn judge(&mut self) -> Result<()> {
3683        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3684        // agent files with `magi task add` name the run that paid for it. The
3685        // prompt overlay is cloned alongside it because the waves borrow it
3686        // while `self` is mutably borrowed by the node's own bookkeeping.
3687        let run_id = self.state.id.clone();
3688        let prompts = self.state.config.prompts.clone();
3689        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3690            return Ok(());
3691        }
3692        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3693        if viable.len() == 1 {
3694            // Recorded so this is a one-time event: `judgements` stays empty
3695            // either way, which without this flag is indistinguishable from
3696            // "not yet judged" on the next reentry — and status is left
3697            // untouched, so a later node's conclusion (e.g. `Blocked` after
3698            // the review budget ran out) survives a resume instead of being
3699            // clobbered back to `Judging` by this node running again.
3700            self.state.judge_skipped = true;
3701            self.state.event(
3702                "judge",
3703                format!(
3704                    "only candidate {} produced a change; judging skipped",
3705                    viable[0].label
3706                ),
3707            );
3708            self.state.save()?;
3709            return Ok(());
3710        }
3711        self.state.status = RunStatus::Judging;
3712
3713        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3714        let language = self.state.config.graph.language.clone();
3715        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3716        let sessions = self.state.config.graph.sessions;
3717        let artifacts = agent::artifacts_dir(&self.state.dir());
3718        let root = self.state.worktree_root();
3719        let base_short = short(&self.state.base_commit);
3720
3721        let mut jobs = Vec::new();
3722        let mut orders = Vec::new();
3723        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3724            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3725            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3726            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3727            let seat_key = format!("judge-{}", j + 1);
3728            let seat = self.seat(&seat_key, &spec.id);
3729            jobs.push(SeatJob {
3730                prompt: prompt::judge(
3731                    &self.state.instruction,
3732                    &views,
3733                    self.roles.judges.len(),
3734                    &base_short,
3735                    &language,
3736                ),
3737                spec,
3738                seat,
3739                cwd: root.join(format!("judge-{}", j + 1)),
3740                timeout,
3741                allow_write: false,
3742                sessions,
3743                artifacts: artifacts.clone(),
3744                stem: format!("judge-{}", j + 1),
3745                handover: None,
3746            });
3747        }
3748
3749        self.state.event(
3750            "judge",
3751            format!(
3752                "{} judges ranking {} candidates blind",
3753                jobs.len(),
3754                viable.len()
3755            ),
3756        );
3757        let labels_for_check = labels.clone();
3758        let mut quota_losses = Vec::new();
3759        let cache = self.state.config.cache_dir();
3760        let ctx = WaveCtx {
3761            carry_seats: false,
3762            pauses: Some((&self.pause, &self.interrupt)),
3763            run: &run_id,
3764            node: "judge",
3765            prompts: &prompts,
3766            cache: cache.as_deref(),
3767            round: None,
3768        };
3769        let results = ask_json_wave::<Ranking>(
3770            jobs,
3771            Arc::clone(&self.sem),
3772            self.state.config.graph.retries,
3773            &self.roles.judge_roster,
3774            &ctx,
3775            &mut quota_losses,
3776            &mut self.state,
3777            &move |r: &Ranking| r.validate(&labels_for_check),
3778        )
3779        .await;
3780        if self.state.resource_block.is_some() {
3781            return self.stop_for_resource();
3782        }
3783        self.state.quota.extend(quota_losses);
3784
3785        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3786            let agent_id = seat.agent.clone();
3787            self.state.seats.insert(seat.key.clone(), seat);
3788            let mut record = Judgement {
3789                judge: j + 1,
3790                seat: format!("judge-{}", j + 1),
3791                agent: agent_id,
3792                ranking: Vec::new(),
3793                reasons: BTreeMap::new(),
3794                confidence: None,
3795                strengths: BTreeMap::new(),
3796                order: orders[j].clone(),
3797                failed: None,
3798                duration_ms: 0,
3799            };
3800            match res {
3801                Ok((ranking, out)) => {
3802                    record.ranking = ranking.normalized();
3803                    record.reasons = ranking.reasons;
3804                    record.strengths = ranking.strengths;
3805                    record.confidence = ranking.confidence;
3806                    record.duration_ms = out.duration_ms;
3807                    self.state.event(
3808                        "judge",
3809                        format!(
3810                            "judge {} ranked {}",
3811                            j + 1,
3812                            record.ranking.iter().collect::<String>()
3813                        ),
3814                    );
3815                }
3816                Err(e) => {
3817                    record.failed = Some(e.to_string());
3818                    self.state
3819                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3820                }
3821            }
3822            self.state.judgements.push(record);
3823            self.state.save()?;
3824        }
3825        Ok(())
3826    }
3827
3828    // ---------------------------------------------------------- deliberate
3829
3830    async fn deliberate(&mut self) -> Result<()> {
3831        if self.hold_base_sync().await? {
3832            return Ok(());
3833        }
3834        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3835        // agent files with `magi task add` name the run that paid for it. The
3836        // prompt overlay is cloned alongside it because the waves borrow it
3837        // while `self` is mutably borrowed by the node's own bookkeeping.
3838        let run_id = self.state.id.clone();
3839        let prompts = self.state.config.prompts.clone();
3840        let rounds = self.state.config.graph.deliberate_rounds;
3841        // Resume unfinished rounds after an outage, but never reopen a tally.
3842        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
3843            return Ok(());
3844        }
3845        let tops: Vec<char> = self
3846            .state
3847            .judgements
3848            .iter()
3849            .filter_map(|j| j.ranking.first().copied())
3850            .collect();
3851        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3852            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3853                self.state.event(
3854                    "deliberate",
3855                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3856                );
3857            }
3858            self.state.status = RunStatus::Voting;
3859            self.state.save()?;
3860            return Ok(());
3861        }
3862
3863        self.state.status = RunStatus::Deliberating;
3864        self.state.event(
3865            "deliberate",
3866            format!(
3867                "split: first choices were {} — opening {rounds} round(s)",
3868                tops.iter().collect::<String>()
3869            ),
3870        );
3871
3872        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3873        let language = self.state.config.graph.language.clone();
3874        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3875        let sessions = self.state.config.graph.sessions;
3876        let artifacts = agent::artifacts_dir(&self.state.dir());
3877        let root = self.state.worktree_root();
3878        let base_short = short(&self.state.base_commit);
3879
3880        // Judges argue in sequence so that a turn can answer the one before it;
3881        // that is the difference between deliberation and three parallel
3882        // monologues.
3883        for round in self.state.deliberation.len() + 1..=rounds {
3884            let mut turns: Vec<DeliberationTurn> = Vec::new();
3885            let mut round_losses = Vec::new();
3886            let mut asked = 0;
3887            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3888                if self.state.judgements[j].failed.is_some() {
3889                    continue;
3890                }
3891                let seat_key = format!("judge-{}", j + 1);
3892                let spec = self.occupant(&seat_key, spec);
3893                let mut seat = self.seat(&seat_key, &spec.id);
3894                let transcript = self.transcript(&turns, j);
3895                let build = |context: Option<&str>| {
3896                    prompt::deliberate(
3897                        &self.state.instruction,
3898                        context,
3899                        &transcript,
3900                        round,
3901                        rounds,
3902                        &language,
3903                    )
3904                };
3905                let block = self.candidate_block(&viable, &base_short);
3906                let full = build(Some(&block));
3907                let text = if has_context(&spec, &seat, sessions) {
3908                    build(None)
3909                } else {
3910                    full.clone()
3911                };
3912                let job = SeatJob {
3913                    spec,
3914                    seat: seat.clone(),
3915                    prompt: text,
3916                    cwd: root.join(format!("judge-{}", j + 1)),
3917                    timeout,
3918                    allow_write: false,
3919                    sessions,
3920                    artifacts: artifacts.clone(),
3921                    stem: format!("delib-{round}-judge-{}", j + 1),
3922                    handover: Some(full),
3923                };
3924                let cache = self.state.config.cache_dir();
3925                let ctx = WaveCtx {
3926                    carry_seats: false,
3927                    pauses: Some((&self.pause, &self.interrupt)),
3928                    run: &run_id,
3929                    node: "deliberate",
3930                    prompts: &prompts,
3931                    cache: cache.as_deref(),
3932                    round: None,
3933                };
3934                // A turn is never nudged (`retries` 0): a failed seat is
3935                // handed to the next roster agent, which gets the full
3936                // context. An empty answer is a turn, not a failure.
3937                let mut losses = Vec::new();
3938                asked += 1;
3939                let mut results = ask_wave_with::<String>(
3940                    vec![job],
3941                    Arc::clone(&self.sem),
3942                    0,
3943                    &self.roles.judge_roster,
3944                    &ctx,
3945                    &mut losses,
3946                    &mut self.state,
3947                    &|text: &str| {
3948                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3949                    },
3950                )
3951                .await;
3952                if self.state.resource_block.is_some() {
3953                    return self.stop_for_resource();
3954                }
3955                round_losses.extend(losses.iter().cloned());
3956                self.state.quota.extend(losses);
3957                let (updated, res, _) = results.pop().expect("one job in, one result out");
3958                seat = updated;
3959                let agent_id = seat.agent.clone();
3960                self.state.seats.insert(seat.key.clone(), seat);
3961                let body = match res {
3962                    Ok((body, _)) => body,
3963                    // Skip the seat; a CLI's raw error JSON is never read as
3964                    // this judge's position.
3965                    Err(e) => {
3966                        self.state
3967                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3968                        continue;
3969                    }
3970                };
3971                let tentative = verdict::extract_json::<Position>(&body)
3972                    .ok()
3973                    .and_then(|p| p.tentative)
3974                    .and_then(|s| s.trim().chars().next())
3975                    .map(|c| c.to_ascii_uppercase());
3976                self.state.event(
3977                    "deliberate",
3978                    format!(
3979                        "round {round}: judge {} now favours {}",
3980                        j + 1,
3981                        tentative.map_or("—".to_owned(), |c| c.to_string())
3982                    ),
3983                );
3984                turns.push(DeliberationTurn {
3985                    judge: j + 1,
3986                    agent: agent_id,
3987                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3988                    tentative,
3989                });
3990            }
3991            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
3992                block_machine_outage(&mut self.state, "deliberate", &round_losses);
3993                return self.stop_for_resource();
3994            }
3995            self.state
3996                .deliberation
3997                .push(DeliberationRound { round, turns });
3998            self.state.save()?;
3999        }
4000
4001        self.state.status = RunStatus::Voting;
4002        self.state.save()?;
4003        Ok(())
4004    }
4005
4006    // ---------------------------------------------------------------- vote
4007
4008    async fn vote(&mut self) -> Result<()> {
4009        if self.hold_base_sync().await? {
4010            return Ok(());
4011        }
4012        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4013        // agent files with `magi task add` name the run that paid for it. The
4014        // prompt overlay is cloned alongside it because the waves borrow it
4015        // while `self` is mutably borrowed by the node's own bookkeeping.
4016        let run_id = self.state.id.clone();
4017        let prompts = self.state.config.prompts.clone();
4018        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4019            return Ok(());
4020        }
4021        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4022        if viable.len() == 1 {
4023            return Ok(());
4024        }
4025        self.state.status = RunStatus::Voting;
4026
4027        let language = self.state.config.graph.language.clone();
4028        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4029        let sessions = self.state.config.graph.sessions;
4030        let artifacts = agent::artifacts_dir(&self.state.dir());
4031        let root = self.state.worktree_root();
4032        let base_short = short(&self.state.base_commit);
4033        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4034
4035        let mut jobs = Vec::new();
4036        let mut seats_at = Vec::new();
4037        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4038            if self
4039                .state
4040                .judgements
4041                .get(j)
4042                .is_some_and(|r| r.failed.is_some())
4043            {
4044                continue;
4045            }
4046            let seat_key = format!("judge-{}", j + 1);
4047            let spec = self.occupant(&seat_key, spec);
4048            let seat = self.seat(&seat_key, &spec.id);
4049            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4050            let text = if has_context(&spec, &seat, sessions) {
4051                prompt::final_vote(&viable, &language)
4052            } else {
4053                full.clone()
4054            };
4055            jobs.push(SeatJob {
4056                spec,
4057                seat,
4058                prompt: text,
4059                cwd: root.join(format!("judge-{}", j + 1)),
4060                timeout,
4061                allow_write: false,
4062                sessions,
4063                artifacts: artifacts.clone(),
4064                stem: format!("vote-judge-{}", j + 1),
4065                handover: Some(full),
4066            });
4067            seats_at.push(j);
4068        }
4069
4070        self.state.event(
4071            "vote",
4072            format!(
4073                "collecting {} final votes one by one, privately",
4074                jobs.len()
4075            ),
4076        );
4077        let allowed = viable.clone();
4078        let mut quota_losses = Vec::new();
4079        let cache = self.state.config.cache_dir();
4080        let ctx = WaveCtx {
4081            carry_seats: false,
4082            pauses: Some((&self.pause, &self.interrupt)),
4083            run: &run_id,
4084            node: "vote",
4085            prompts: &prompts,
4086            cache: cache.as_deref(),
4087            round: None,
4088        };
4089        let results = ask_json_wave::<FinalVote>(
4090            jobs,
4091            Arc::clone(&self.sem),
4092            self.state.config.graph.retries,
4093            &self.roles.judge_roster,
4094            &ctx,
4095            &mut quota_losses,
4096            &mut self.state,
4097            &move |v: &FinalVote| match v.label() {
4098                Some(c) if allowed.contains(&c) => Ok(()),
4099                other => bail!("vote {other:?} is not one of {allowed:?}"),
4100            },
4101        )
4102        .await;
4103        if self.state.resource_block.is_some() {
4104            return self.stop_for_resource();
4105        }
4106        self.state.quota.extend(quota_losses);
4107
4108        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4109            let agent_id = seat.agent.clone();
4110            self.state.seats.insert(seat.key.clone(), seat);
4111            let initial = self
4112                .state
4113                .judgements
4114                .get(j)
4115                .and_then(|r| r.ranking.first().copied());
4116            let mut record = VoteRecord {
4117                judge: j + 1,
4118                agent: agent_id,
4119                vote: None,
4120                reason: String::new(),
4121                changed: false,
4122            };
4123            match res {
4124                Ok((v, _)) => {
4125                    record.vote = v.label();
4126                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4127                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4128                    self.state.event(
4129                        "vote",
4130                        format!(
4131                            "judge {} voted {}{}",
4132                            j + 1,
4133                            record.vote.unwrap_or('?'),
4134                            if record.changed { " (changed)" } else { "" }
4135                        ),
4136                    );
4137                }
4138                Err(e) => {
4139                    self.state
4140                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4141                }
4142            }
4143            self.state.votes.push(record);
4144            self.state.save()?;
4145        }
4146        Ok(())
4147    }
4148
4149    // --------------------------------------------------------------- tally
4150
4151    fn tally(&mut self) -> Result<()> {
4152        if self.state.tally.is_some() {
4153            return Ok(());
4154        }
4155        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4156        let tops: Vec<char> = self
4157            .state
4158            .judgements
4159            .iter()
4160            .filter_map(|j| j.ranking.first().copied())
4161            .collect();
4162        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4163
4164        // A judge whose private vote failed still counted once, in the initial
4165        // ranking; using it beats discarding a whole seat.
4166        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4167        let mut cast: Vec<char> = Vec::new();
4168        for (i, j) in self.state.judgements.iter().enumerate() {
4169            let vote = self
4170                .state
4171                .votes
4172                .iter()
4173                .find(|v| v.judge == i + 1)
4174                .and_then(|v| v.vote)
4175                .or_else(|| j.ranking.first().copied());
4176            if let Some(v) = vote {
4177                *first_choice.entry(v).or_insert(0) += 1;
4178                cast.push(v);
4179            }
4180        }
4181
4182        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4183        for j in &self.state.judgements {
4184            let n = j.ranking.len();
4185            for (pos, label) in j.ranking.iter().enumerate() {
4186                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4187            }
4188        }
4189
4190        let best = first_choice.values().copied().max().unwrap_or(0);
4191        let mut leaders: Vec<char> = first_choice
4192            .iter()
4193            .filter(|(_, v)| **v == best)
4194            .map(|(k, _)| *k)
4195            .collect();
4196        let mut tie_break = None;
4197        if leaders.len() > 1 {
4198            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4199            let borda_leaders: Vec<char> = leaders
4200                .iter()
4201                .copied()
4202                .filter(|l| borda[l] == top_borda)
4203                .collect();
4204            tie_break = Some(if borda_leaders.len() == 1 {
4205                format!(
4206                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4207                    leaders.len()
4208                )
4209            } else {
4210                format!(
4211                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4212                    leaders.len()
4213                )
4214            });
4215            leaders = borda_leaders;
4216            leaders.sort_unstable();
4217        }
4218        let winner = *leaders
4219            .first()
4220            .or(viable.first())
4221            .context("no candidate to declare a winner from")?;
4222
4223        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4224        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4225        let deliberated = !self.state.deliberation.is_empty();
4226
4227        // Whose verdict is this? A rate-limited seat is absent even if it
4228        // ranked before the limit hit, so presence is measured against the
4229        // recorded losses, not just "did a ranking ever appear".
4230        let quota_seats: std::collections::BTreeSet<&str> =
4231            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4232        let mut present = 0usize;
4233        for (i, j) in self.state.judgements.iter().enumerate() {
4234            if quota_seats.contains(j.seat.as_str()) {
4235                continue;
4236            }
4237            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4238            let voted = self
4239                .state
4240                .votes
4241                .iter()
4242                .any(|v| v.judge == i + 1 && v.vote.is_some());
4243            if ranked || voted {
4244                present += 1;
4245            }
4246        }
4247        // Strict majority of the configured panel. A bare majority is real
4248        // signal we can act on, while a minority verdict must never stand in
4249        // for a healthy one. A one-candidate run needs no panel at all, and
4250        // `judges` stays `0` rather than the roster size a panel that never
4251        // sat would otherwise be credited with.
4252        let needs_quorum = viable.len() > 1;
4253        let judges_total = if needs_quorum {
4254            self.roles.judges.len()
4255        } else {
4256            0
4257        };
4258        let quorum = if needs_quorum {
4259            judges_total / 2 + 1
4260        } else {
4261            0
4262        };
4263        let met_quorum = !needs_quorum || present >= quorum;
4264        let uncontested = (!needs_quorum).then(|| {
4265            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4266        });
4267
4268        self.state.event(
4269            "tally",
4270            match &uncontested {
4271                Some(reason) => format!("winner {winner} — {reason}"),
4272                None => format!(
4273                    "winner {winner} — votes {} | initial {} | {} changed | \
4274                     {present}/{judges_total} judges{}",
4275                    first_choice
4276                        .iter()
4277                        .map(|(k, v)| format!("{k}:{v}"))
4278                        .collect::<Vec<_>>()
4279                        .join(" "),
4280                    if unanimous_initial {
4281                        "unanimous"
4282                    } else {
4283                        "split"
4284                    },
4285                    changed_votes,
4286                    if met_quorum {
4287                        String::new()
4288                    } else {
4289                        format!(" — below quorum ({quorum} required)")
4290                    },
4291                ),
4292            },
4293        );
4294        if !met_quorum {
4295            self.state.event(
4296                "stall",
4297                format!(
4298                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4299                     the run stops here, resumable"
4300                ),
4301            );
4302        }
4303        self.state.tally = Some(Tally {
4304            first_choice,
4305            borda,
4306            winner,
4307            rankings: tops.len(),
4308            unanimous_initial,
4309            deliberated,
4310            changed_votes,
4311            unanimous_final,
4312            tie_break,
4313            judges: judges_total,
4314            present,
4315            quorum,
4316            met_quorum,
4317            uncontested,
4318        });
4319        self.state.status = if met_quorum {
4320            RunStatus::Reviewing
4321        } else {
4322            RunStatus::Stalled
4323        };
4324        self.state.save()?;
4325        Ok(())
4326    }
4327
4328    // ------------------------------------------------------------- recover
4329
4330    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4331    /// resumed toward completion once the transient cause clears.
4332    ///
4333    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4334    /// it toward the quorum, which is exactly the set of seats whose absence
4335    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4336    /// not depend on which node happened to hit the limit), or an ordinary
4337    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4338    /// seat is never disturbed.
4339    ///
4340    /// A seat that now answers with a usable ranking is "recovered": its
4341    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4342    /// `tally` counts it present again), and its vote re-collected. A seat that
4343    /// still fails keeps its loss and stays absent.
4344    ///
4345    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4346    /// to review/gate/merge), `false` when it is still below quorum (the run
4347    /// stays `Stalled`, still resumable for a later retry).
4348    #[allow(clippy::too_many_lines)]
4349    async fn recover_stall(&mut self) -> Result<bool> {
4350        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4351        // agent files with `magi task add` name the run that paid for it. The
4352        // prompt overlay is cloned alongside it because the waves borrow it
4353        // while `self` is mutably borrowed by the node's own bookkeeping.
4354        let run_id = self.state.id.clone();
4355        let prompts = self.state.config.prompts.clone();
4356        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4357        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4358        // is what keeps a non-quota collapse — or a quota loss recorded at the
4359        // deliberate node — from being a permanent dead-end on `--resume`.
4360        let quota_seats: BTreeSet<&str> =
4361            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4362        let absent: Vec<String> = self
4363            .state
4364            .judgements
4365            .iter()
4366            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4367            .map(|j| j.seat.clone())
4368            .collect();
4369        if absent.is_empty() {
4370            return Ok(false);
4371        }
4372        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4373        if viable.len() <= 1 {
4374            return Ok(false);
4375        }
4376        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4377        let language = self.state.config.graph.language.clone();
4378        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4379        let sessions = self.state.config.graph.sessions;
4380        let artifacts = agent::artifacts_dir(&self.state.dir());
4381        let root = self.state.worktree_root();
4382        let base_short = short(&self.state.base_commit);
4383        let candidates: Vec<Candidate> = viable.clone();
4384
4385        // Map each absent seat key to its 0-based position in `roles.judges`.
4386        let mut positions: Vec<usize> = absent
4387            .iter()
4388            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4389            .collect();
4390        if positions.is_empty() {
4391            return Ok(false);
4392        }
4393        positions.sort_unstable();
4394        positions.dedup();
4395
4396        // Re-rank the lost seats, one blind prompt each.
4397        let mut judge_jobs = Vec::new();
4398        for &j in &positions {
4399            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4400            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4401            let seat_key = format!("judge-{}", j + 1);
4402            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4403            let seat = self.seat(&seat_key, &spec.id);
4404            judge_jobs.push(SeatJob {
4405                spec,
4406                seat,
4407                prompt: prompt::judge(
4408                    &self.state.instruction,
4409                    &views,
4410                    self.roles.judges.len(),
4411                    &base_short,
4412                    &language,
4413                ),
4414                cwd: root.join(seat_key),
4415                timeout,
4416                allow_write: false,
4417                sessions,
4418                artifacts: artifacts.clone(),
4419                stem: format!("judge-{}-recover", j + 1),
4420                handover: None,
4421            });
4422        }
4423
4424        let labels_for_check = labels.clone();
4425        let mut judge_losses = Vec::new();
4426        let retries = self.state.config.graph.retries;
4427        let cache = self.state.config.cache_dir();
4428        let ctx = WaveCtx {
4429            carry_seats: false,
4430            pauses: Some((&self.pause, &self.interrupt)),
4431            run: &run_id,
4432            node: "judge",
4433            prompts: &prompts,
4434            cache: cache.as_deref(),
4435            round: None,
4436        };
4437        let results = ask_json_wave::<Ranking>(
4438            judge_jobs,
4439            Arc::clone(&self.sem),
4440            retries,
4441            &self.roles.judge_roster,
4442            &ctx,
4443            &mut judge_losses,
4444            &mut self.state,
4445            &move |r: &Ranking| r.validate(&labels_for_check),
4446        )
4447        .await;
4448        if self.state.resource_block.is_some() {
4449            self.stop_for_resource()?;
4450            return Ok(false);
4451        }
4452
4453        // Refresh the judgement of every seat that ranked again.
4454        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4455        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4456            let agent_id = seat.agent.clone();
4457            self.state.seats.insert(seat.key.clone(), seat);
4458            let record = &mut self.state.judgements[j];
4459            match res {
4460                Ok((ranking, out)) => {
4461                    record.agent = agent_id;
4462                    record.ranking = ranking.normalized();
4463                    record.reasons = ranking.reasons;
4464                    record.strengths = ranking.strengths;
4465                    record.confidence = ranking.confidence;
4466                    record.failed = None;
4467                    record.duration_ms = out.duration_ms;
4468                    recovered.insert(j);
4469                    self.state.event(
4470                        "recover",
4471                        format!("judge {} ranked again after the limit", j + 1),
4472                    );
4473                }
4474                Err(e) => {
4475                    self.state
4476                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4477                }
4478            }
4479        }
4480
4481        // Re-ask the votes of the seats that recovered a ranking.
4482        let mut vote_jobs = Vec::new();
4483        let mut vote_pos: Vec<usize> = Vec::new();
4484        for &j in &recovered {
4485            let seat_key = format!("judge-{}", j + 1);
4486            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4487            let seat = self.seat(&seat_key, &spec.id);
4488            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4489            let text = if has_context(&spec, &seat, sessions) {
4490                prompt::final_vote(&labels, &language)
4491            } else {
4492                full.clone()
4493            };
4494            vote_jobs.push(SeatJob {
4495                spec,
4496                seat,
4497                prompt: text,
4498                cwd: root.join(seat_key),
4499                timeout,
4500                allow_write: false,
4501                sessions,
4502                artifacts: artifacts.clone(),
4503                stem: format!("vote-judge-{}-recover", j + 1),
4504                handover: Some(full),
4505            });
4506            vote_pos.push(j);
4507        }
4508        let allowed = labels.clone();
4509        let mut vote_losses = Vec::new();
4510        let vote_retries = self.state.config.graph.retries;
4511        let vote_cache = self.state.config.cache_dir();
4512        let ctx = WaveCtx {
4513            carry_seats: false,
4514            pauses: Some((&self.pause, &self.interrupt)),
4515            run: &run_id,
4516            node: "vote",
4517            prompts: &prompts,
4518            cache: vote_cache.as_deref(),
4519            round: None,
4520        };
4521        let votes = ask_json_wave::<FinalVote>(
4522            vote_jobs,
4523            Arc::clone(&self.sem),
4524            vote_retries,
4525            &self.roles.judge_roster,
4526            &ctx,
4527            &mut vote_losses,
4528            &mut self.state,
4529            &move |v: &FinalVote| match v.label() {
4530                Some(c) if allowed.contains(&c) => Ok(()),
4531                other => bail!("vote {other:?} is not one of {allowed:?}"),
4532            },
4533        )
4534        .await;
4535        if self.state.resource_block.is_some() {
4536            self.stop_for_resource()?;
4537            return Ok(false);
4538        }
4539        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4540            let agent_id = seat.agent.clone();
4541            self.state.seats.insert(seat.key.clone(), seat);
4542            match res {
4543                Ok((v, _)) => {
4544                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4545                        rec.vote = v.label();
4546                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4547                    } else {
4548                        self.state.votes.push(VoteRecord {
4549                            judge: j + 1,
4550                            agent: agent_id,
4551                            vote: v.label(),
4552                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4553                            changed: false,
4554                        });
4555                    }
4556                    self.state.event(
4557                        "recover",
4558                        format!("judge {} voted again after the limit", j + 1),
4559                    );
4560                }
4561                Err(e) => {
4562                    self.state
4563                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4564                }
4565            }
4566        }
4567
4568        // A seat that ranked again is present even if its re-vote failed —
4569        // `tally` falls back to the initial ranking's first choice — so clear
4570        // its quota loss. Seats that still fail keep theirs and stay absent.
4571        let recovered_keys: BTreeSet<String> = recovered
4572            .iter()
4573            .map(|&j| format!("judge-{}", j + 1))
4574            .collect();
4575        self.state
4576            .quota
4577            .retain(|q| !recovered_keys.contains(&q.seat));
4578        // A seat that hit the limit again is a fresh loss, not the old one:
4579        // replace the stale entry so the history stays one-per-seat and the
4580        // daemon can tell this attempt's loss from a previous session's.
4581        for loss in judge_losses.into_iter().chain(vote_losses) {
4582            if recovered_keys.contains(&loss.seat) {
4583                continue;
4584            }
4585            self.state.quota.retain(|q| q.seat != loss.seat);
4586            self.state.quota.push(loss);
4587        }
4588
4589        // Recompute the verdict from the refreshed panel.
4590        self.state.tally = None;
4591        self.tally()?;
4592        Ok(self
4593            .state
4594            .tally
4595            .as_ref()
4596            .map(|t| t.met_quorum)
4597            .unwrap_or(false))
4598    }
4599
4600    // ---------------------------------------------------------------- graft
4601
4602    /// Port the strengths the judges attributed to the losing candidates onto
4603    /// the winner, once, between the tally and the review loop.
4604    ///
4605    /// Best-effort by construction: it returns nothing and fails nothing. A
4606    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4607    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4608    /// the run carries on as if the phase were off. See [`graft`] for the
4609    /// decisions and `AGENTS.md` for the invariants.
4610    ///
4611    /// The one exception is a rollback that could not be confirmed: the
4612    /// rejected commit may still be the winner's tip, so the run is blocked
4613    /// (resumable; the record stays `Running`, which makes a resume retry the
4614    /// restore) instead of reviewing, gating and merging it.
4615    async fn graft(&mut self) -> Result<()> {
4616        let Err(e) = self.graft_inner().await else {
4617            return Ok(());
4618        };
4619        if e.is::<crate::repo_lock::ResourceBlocked>() {
4620            return Err(e);
4621        }
4622        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4623            self.state.status = RunStatus::Blocked;
4624            self.state.event("graft", format!("blocked: {e:#}"));
4625            self.state.save()?;
4626            return Err(e);
4627        }
4628        // An error after the `Running` record was saved may have left the
4629        // fixer's work in the tree: the record must not stay unsettled
4630        // (a later resume would reset past the review fixes), so confirm the
4631        // restore now, or block.
4632        if let Some(rec) = self.state.graft.clone()
4633            && !rec.outcome.settled()
4634        {
4635            if let Some(winner) = self.state.winner().cloned() {
4636                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4637                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4638                        return Err(re);
4639                    }
4640                    self.state.status = RunStatus::Blocked;
4641                    self.state.event("graft", format!("blocked: {re:#}"));
4642                    self.state.save()?;
4643                    return Err(re);
4644                }
4645            }
4646            if let Some(g) = self.state.graft.as_mut() {
4647                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4648            }
4649        }
4650        self.state.event("graft", format!("skipped: {e:#}"));
4651        if let Err(e) = self.state.save() {
4652            tracing::warn!("save after graft: {e:#}");
4653        }
4654        Ok(())
4655    }
4656
4657    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4658    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4659    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4660        let _coordination = self.coordinate(wt).await?;
4661        crate::recovery::restore_repair(wt, pre)
4662            .await
4663            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4664    }
4665
4666    /// A skip is a run event, recorded once however often the node is re-entered.
4667    fn graft_skipped(&mut self, why: &str) {
4668        let message = format!("skipped: {why}");
4669        if !self
4670            .state
4671            .events
4672            .iter()
4673            .any(|e| e.node == "graft" && e.message == message)
4674        {
4675            self.state.event("graft", message);
4676        }
4677    }
4678
4679    async fn graft_inner(&mut self) -> Result<()> {
4680        // A round that was started and never finished (crash, kill): the
4681        // fixer may have left anything in the tree, so put the winner back.
4682        if let Some(rec) = self.state.graft.clone()
4683            && !rec.outcome.settled()
4684        {
4685            // Reviews exist only if the graft ended without its record being
4686            // settled (a swallowed error): the tree has moved on since, and
4687            // a reset to `pre` would discard the review fixes.
4688            if self.state.reviews.is_empty()
4689                && let Some(winner) = self.state.winner().cloned()
4690            {
4691                self.graft_restore(&winner.worktree, &rec.pre).await?;
4692            }
4693            if rec.outcome == GraftOutcome::WaitingForRestore {
4694                self.state.graft = None;
4695                self.state.save()?;
4696            } else {
4697                if let Some(g) = self.state.graft.as_mut() {
4698                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4699                }
4700                self.state.event(
4701                    "graft",
4702                    "rolled back: the previous round was interrupted; the winner is untouched",
4703                );
4704                self.state.save()?;
4705                return Ok(());
4706            }
4707        }
4708        let plan = match graft::plan(&self.state) {
4709            Ok(plan) => plan,
4710            Err(skip) => {
4711                if skip != graft::Skip::AlreadyDone {
4712                    self.graft_skipped(skip.reason());
4713                    self.state.save()?;
4714                }
4715                return Ok(());
4716            }
4717        };
4718        let Some(winner) = self.state.winner().cloned() else {
4719            return Ok(());
4720        };
4721        let wt = winner.worktree.clone();
4722        if !git::is_clean(&wt).await? {
4723            self.graft_skipped("the winner's worktree has uncommitted changes");
4724            self.state.save()?;
4725            return Ok(());
4726        }
4727        let pre = git::rev_parse(&wt, "HEAD").await?;
4728        // Saved before the fixer is called: a stop from here on is repaired
4729        // by the branch above on the next entry.
4730        self.state.graft = Some(GraftRecord {
4731            pre: pre.clone(),
4732            post: None,
4733            lines: None,
4734            limit: None,
4735            agent: None,
4736            outcome: GraftOutcome::Running,
4737            requested: graft::requested(&plan),
4738            adoption: None,
4739            adoption_error: None,
4740        });
4741        self.state.event(
4742            "graft",
4743            format!(
4744                "porting strengths of candidate(s) {} onto {}",
4745                plan.strengths
4746                    .iter()
4747                    .map(|(l, _, _)| l.to_string())
4748                    .collect::<Vec<_>>()
4749                    .join(", "),
4750                plan.winner
4751            ),
4752        );
4753        self.state.save()?;
4754
4755        let fix_prompt = prompt::graft(
4756            &self.state.instruction,
4757            plan.winner,
4758            &plan.strengths,
4759            &self.state.config.graph.language,
4760        );
4761        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4762        let sessions = self.state.config.graph.sessions;
4763        let artifacts = agent::artifacts_dir(&self.state.dir());
4764        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4765        let (_, seat, out) = self
4766            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4767                SeatJob {
4768                    prompt: fix_prompt.clone(),
4769                    spec,
4770                    seat,
4771                    cwd: wt.clone(),
4772                    timeout,
4773                    allow_write: true,
4774                    sessions,
4775                    artifacts: artifacts.clone(),
4776                    stem: "graft".to_owned(),
4777                    handover: None,
4778                }
4779            })
4780            .await;
4781        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4782            // Preserve the need to restore earlier fallback edits without
4783            // confusing an unstarted seat with an interrupted invocation.
4784            self.state.graft.as_mut().expect("graft record").outcome =
4785                GraftOutcome::WaitingForRestore;
4786            return self.stop_for_resource();
4787        }
4788        let mut why: Option<String> = None;
4789        match &out {
4790            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4791                why = Some("the fixer produced nothing usable".to_owned());
4792            }
4793            AgentOutcome::Ok(_) => {}
4794            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4795            AgentOutcome::Quota(o) => {
4796                self.state.quota.push(QuotaLoss::from_report(
4797                    seat.key.clone(),
4798                    "graft",
4799                    o.quota.as_ref(),
4800                ));
4801                why = Some("rate limited (quota)".to_owned());
4802            }
4803            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4804                why = Some(format!(
4805                    "the fixer failed: {}",
4806                    e.lines()
4807                        .next()
4808                        .unwrap_or("")
4809                        .chars()
4810                        .take(160)
4811                        .collect::<String>()
4812                ));
4813            }
4814        }
4815        let agent_id = seat.agent.clone();
4816        self.state.seats.insert(seat.key.clone(), seat);
4817
4818        let mut post = None;
4819        let mut lines = None;
4820        let _coordination = self.coordinate(&wt).await?;
4821        if why.is_none() {
4822            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4823            if let Ok(r) = &rescued {
4824                self.state.note_withheld("graft", &r.withheld);
4825            }
4826            let head = git::rev_parse(&wt, "HEAD").await?;
4827            // What is measured must be what gets reviewed and gated: a failed
4828            // rescue, or anything still uncommitted (withheld lockfiles
4829            // aside, which are never part of the tree), rejects the graft.
4830            let dirty = match &rescued {
4831                Ok(r) => git::status_porcelain(&wt)
4832                    .await
4833                    .map(|st| {
4834                        st.lines().any(|l| {
4835                            !r.withheld
4836                                .iter()
4837                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4838                        })
4839                    })
4840                    .unwrap_or(true),
4841                Err(_) => true,
4842            };
4843            if dirty {
4844                why = Some("the fixer left changes that could not be committed".to_owned());
4845            } else if head == pre {
4846                why = Some("the fixer changed nothing".to_owned());
4847            } else {
4848                lines = git::changed_lines(&wt, &pre, &head).await;
4849                post = Some(head);
4850            }
4851        }
4852        let (adoption, adoption_error) = match &out {
4853            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4854            _ => (None, None),
4855        };
4856        let requested = graft::requested(&plan);
4857        let outcome = match why {
4858            None => {
4859                self.state.event(
4860                    "graft",
4861                    format!(
4862                        "applied: {}",
4863                        lines.map_or_else(
4864                            || "changed lines unavailable".to_owned(),
4865                            |n| format!("{n} changed lines")
4866                        )
4867                    ),
4868                );
4869                GraftOutcome::Applied
4870            }
4871            Some(why) => {
4872                // Not settled until the restore is confirmed: the record
4873                // stays `Running`, so a resume retries it.
4874                crate::recovery::restore_repair(&wt, &pre)
4875                    .await
4876                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4877                self.state.event(
4878                    "graft",
4879                    format!("rolled back: {why}; the winner is untouched"),
4880                );
4881                GraftOutcome::RolledBack(why)
4882            }
4883        };
4884        self.state.graft = Some(GraftRecord {
4885            pre,
4886            post,
4887            lines,
4888            limit: None,
4889            agent: Some(agent_id),
4890            outcome,
4891            requested,
4892            adoption,
4893            adoption_error,
4894        });
4895        self.state.save()?;
4896        Ok(())
4897    }
4898
4899    // ----------------------------------------------------------------- fold
4900
4901    async fn fold_losers(&mut self) -> Result<()> {
4902        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4903            return Ok(());
4904        };
4905        if !self.state.candidate_preservation.is_empty() {
4906            bail!("candidate preservation incomplete; refusing cleanup");
4907        }
4908        let repo = self.state.repo.clone();
4909        let _coordination = self.coordinate(&repo).await?;
4910        let mut folded = Vec::new();
4911        for i in 0..self.state.candidates.len() {
4912            let c = &self.state.candidates[i];
4913            if c.label == winner || c.folded {
4914                continue;
4915            }
4916            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4917            remove_preserved_worktree(&repo, &wt).await?;
4918            if git::branch_exists(&repo, &branch).await?
4919                && !git::branch_delete(&repo, &branch).await?
4920            {
4921                bail!("candidate branch removal refused");
4922            }
4923            self.state.candidates[i].folded = true;
4924            self.state.save()?;
4925            folded.push(label.to_string());
4926        }
4927        // The judges are finished; their checkouts are pure cost from here.
4928        let root = self.state.worktree_root();
4929        for j in 1..=self.roles.judges.len() {
4930            let wt = root.join(format!("judge-{j}"));
4931            if wt.exists() {
4932                remove_preserved_worktree(&repo, &wt).await?;
4933            }
4934        }
4935        // The design-deliberation stage is finished by the time a tally
4936        // exists — same reasoning as the judges above.
4937        if self.state.config.graph.advise {
4938            for k in 1..=self.state.config.graph.advisors {
4939                let wt = root.join(format!("advisor-{k}"));
4940                if wt.exists() {
4941                    remove_preserved_worktree(&repo, &wt).await?;
4942                }
4943            }
4944        }
4945        if !folded.is_empty() {
4946            self.state
4947                .event("fold", format!("folded candidates {}", folded.join(", ")));
4948            self.state.save()?;
4949        }
4950        Ok(())
4951    }
4952
4953    // ------------------------------------------------------------ base sync
4954
4955    async fn hold_base_sync(&mut self) -> Result<bool> {
4956        self.hold_base_sync_as(None).await
4957    }
4958
4959    async fn hold_base_sync_kind(
4960        &mut self,
4961        kind: crate::base_sync_recovery::StopKind,
4962    ) -> Result<()> {
4963        self.hold_base_sync_as(Some(kind)).await?;
4964        Ok(())
4965    }
4966
4967    async fn hold_base_sync_as(
4968        &mut self,
4969        kind: Option<crate::base_sync_recovery::StopKind>,
4970    ) -> Result<bool> {
4971        if !self.state.base_sync_blocked() {
4972            self.state.base_sync_recovery.stop = None;
4973            return Ok(false);
4974        }
4975        let head = match self.state.winner() {
4976            Some(w) => git::rev_parse(&w.worktree, "HEAD")
4977                .await
4978                .unwrap_or_default(),
4979            None => String::new(),
4980        };
4981        self.state.record_base_sync_stop(&head);
4982        if let Some(kind) = kind {
4983            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
4984        }
4985        self.state.save()?;
4986        Ok(true)
4987    }
4988
4989    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4990    /// anything verifies it.
4991    ///
4992    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4993    /// read whatever is checked out in the winner's worktree. Left alone that
4994    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4995    /// it when the run *branched* - and a run takes long enough that the base
4996    /// has usually moved by the time it gets here. A gate that ran there
4997    /// answers "green on the commit this run started from", not "green on
4998    /// what is about to land", and the difference showed up three times in
4999    /// one day as a green run whose merge would have reverted a file another
5000    /// pull request had already landed.
5001    ///
5002    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5003    /// `land::Step::Rebase` calls, rather than a second implementation of the
5004    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5005    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5006    /// doc warns about ("two answers to a question nobody notices until a
5007    /// diff is wrong").
5008    ///
5009    /// Foreign-checkout reviews instead merge the base into the owned shadow
5010    /// worktree, preserving the ancestry required for fast-forward delivery.
5011    /// A conflicting merge is aborted and held for operator resolution.
5012    ///
5013    /// A conflict is not the end of the road: the standing rebase is handed
5014    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5015    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5016    /// land). Once it finishes, review and the gate run as usual on the
5017    /// rebased tree, which is where a breakage the new base caused is caught
5018    /// by the ordinary gate-fix round. magi resolves nothing itself.
5019    ///
5020    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5021    /// `state.base_sync.attempts`, is how many times the base is *rebased
5022    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
5023    /// *conflict* was given to a fixer. When the fixer cannot finish the
5024    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5025    /// what was tried (rounds spent, paths still conflicted) and the branch
5026    /// and worktree stay exactly as they were - untouched, for a person to
5027    /// look at - which is also what makes re-entering this function
5028    /// afterwards a no-op instead of a second attempt at the same wall. A
5029    /// push failure ends the same way.
5030    async fn sync_to_base(&mut self) -> Result<()> {
5031        if self.state.status == RunStatus::AlreadyInBase {
5032            return Ok(());
5033        }
5034        self.hold_base_sync().await?;
5035        let conflicted = self
5036            .state
5037            .base_sync
5038            .as_ref()
5039            .is_some_and(|s| s.conflict.is_some());
5040        let Some(winner) = self.state.winner().cloned() else {
5041            return Ok(());
5042        };
5043
5044        let repo = self.state.repo.clone();
5045        let remote = self.state.config.merge.remote.clone();
5046        let base_branch = self.state.base_branch.clone();
5047        let tracking = format!("{remote}/{base_branch}");
5048
5049        let fetched_base =
5050            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5051        // No network, or the remote never had this branch: the run already
5052        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5053        // otherwise), and one that got this far is not blocked by a fetch
5054        // that fails now. It just has no newer tip to compare against.
5055        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
5056            return Ok(());
5057        };
5058
5059        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5060        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5061        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5062
5063        // Before any rebase, and before a recorded conflict is honoured: a
5064        // branch whose change reached the base under other commit ids has
5065        // nothing to rebase and nothing to conflict with, and a run that
5066        // already stopped on that phantom conflict recovers here on resume.
5067        // `behind == 0` with head == tip is a branch the base has since taken
5068        // in whole, whether or not a conflict was ever recorded: the ancestry
5069        // proof must still run (`classify` ignores a head still on the start
5070        // commit).
5071        if (behind > 0 || conflicted || head == tip)
5072            && self
5073                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5074                .await?
5075        {
5076            return Ok(());
5077        }
5078        if conflicted && self.state.base_recovery_pending() {
5079            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5080            let owner =
5081                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5082            if !fetched_base
5083                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5084                || !git::is_clean(&winner.worktree).await?
5085                || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
5086            {
5087                return Ok(());
5088            }
5089        }
5090        let recovery = conflicted && self.state.consume_base_recovery(&head);
5091        if conflicted {
5092            // Consumption (including a refused changed HEAD) is durable before Git.
5093            self.state.save()?;
5094            if !recovery {
5095                return Ok(());
5096            }
5097        }
5098
5099        if behind == 0 {
5100            // A fixer-finished rebase moves the branch ref before the
5101            // winner's worktree is told (`sync_to_head` below). A run that
5102            // died in between resumes here with `behind == 0` and a tree still
5103            // holding the pre-rebase files, which review and the gate would
5104            // then read. That state is exactly: HEAD moved off the tip the
5105            // rebase started from, yet the tree is still identical to that
5106            // tip. A tree with edits of its own differs from it, so nothing
5107            // is thrown away.
5108            if let Some(from) = self
5109                .state
5110                .rebase_fixes
5111                .iter()
5112                .rev()
5113                .find_map(|r| r.from.clone())
5114                && from != head
5115                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5116                    .await
5117                    .is_ok_and(|o| o.ok())
5118            {
5119                git::sync_to_head(&winner.worktree).await?;
5120            }
5121            // An earlier attempt may have rebased the branch locally and died
5122            // before pushing it (only the fresh-rebase arm below pushes).
5123            // Publish it now, so the plain push at PR time is not refused as
5124            // a non-fast-forward. A run already holding a recorded conflict
5125            // never reaches here; that case is out of scope.
5126            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5127            if let Some(why) = &conflict {
5128                self.state.status = RunStatus::Blocked;
5129                self.state.event("land", why.clone());
5130            }
5131            self.state.base_sync = Some(BaseSync {
5132                tip,
5133                behind: 0,
5134                attempts: attempts + usize::from(recovery),
5135                conflict,
5136                already_in: None,
5137            });
5138            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5139                .await?;
5140            self.state.save()?;
5141            return Ok(());
5142        }
5143
5144        if attempts >= BASE_SYNC_ROUNDS && !recovery {
5145            let why = format!(
5146                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5147                 synchronization(s); synchronizing again would only race it",
5148                winner.branch
5149            );
5150            self.state.status = RunStatus::Blocked;
5151            self.state.base_sync = Some(BaseSync {
5152                tip,
5153                behind,
5154                attempts,
5155                conflict: Some(why.clone()),
5156                already_in: None,
5157            });
5158            self.state.event("land", why);
5159            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5160                .await?;
5161            self.state.save()?;
5162            return Ok(());
5163        }
5164
5165        if self.state.foreign_checkout.is_some() {
5166            // Delivery to the operator's checkout is fast-forward only. A
5167            // rebase here would discard its saved head from our ancestry.
5168            let wait = repository_wait(&self.state, None);
5169            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5170            let merged = git::git_raw(
5171                &winner.worktree,
5172                &[
5173                    "-c",
5174                    "core.hooksPath=/dev/null",
5175                    "merge",
5176                    "--no-ff",
5177                    "--no-autostash",
5178                    "--no-edit",
5179                    "-m",
5180                    &format!("Merge {tracking} before verifying"),
5181                    &tip,
5182                ],
5183            )
5184            .await?;
5185            let conflict = if merged.ok() {
5186                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5187                self.state.event(
5188                    "land",
5189                    format!("merged {tracking} into {} before verifying", winner.branch),
5190                );
5191                None
5192            } else {
5193                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5194                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5195                }
5196                let why = format!(
5197                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5198                    winner.branch,
5199                    merged.stderr.chars().take(600).collect::<String>()
5200                );
5201                self.state.status = RunStatus::Blocked;
5202                self.state.event("land", &why);
5203                Some(why)
5204            };
5205            let synced = conflict.is_none();
5206            self.state.base_sync = Some(BaseSync {
5207                tip,
5208                behind: if synced { 0 } else { behind },
5209                attempts: attempts + 1,
5210                conflict,
5211                already_in: None,
5212            });
5213            self.state.save()?;
5214            return Ok(());
5215        }
5216
5217        self.state.event(
5218            "land",
5219            format!(
5220                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5221                winner.branch
5222            ),
5223        );
5224        self.state.save()?;
5225
5226        // The remote's copy of the branch, read now and only if the fetch
5227        // really succeeded (a stale tracking ref must never pin a lease). It is
5228        // pushed over after a rebase only when it is a commit this branch
5229        // already contains, by ancestry or by patch (an earlier rebase of ours
5230        // that never reached the remote): anything else is somebody else's work.
5231        let branch_tracking = format!("{remote}/{}", winner.branch);
5232        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5233        let remote_tip = if self.state.task_pr_branch().is_none()
5234            && matches!(&fetched_branch, Ok(o) if o.ok())
5235        {
5236            git::rev_parse(&repo, &branch_tracking).await.ok()
5237        } else {
5238            None
5239        };
5240        // A remote tip this branch does not contain is somebody else's work:
5241        // rebasing would leave a local tip that can never be pushed. Stop
5242        // before touching anything and say so.
5243        if let Some(theirs) = &remote_tip
5244            && !git::is_ancestor(&repo, theirs, &head).await
5245            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5246                .await
5247                .is_ok_and(|missing| missing.is_empty())
5248        {
5249            let why = format!(
5250                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5251                 them",
5252                short(theirs),
5253                winner.branch
5254            );
5255            self.state.status = RunStatus::Blocked;
5256            self.state.base_sync = Some(BaseSync {
5257                tip,
5258                behind,
5259                attempts: attempts + usize::from(recovery),
5260                conflict: Some(why.clone()),
5261                already_in: None,
5262            });
5263            self.state.event("land", why);
5264            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5265                .await?;
5266            self.state.save()?;
5267            return Ok(());
5268        }
5269
5270        let scratch = self.state.dir().join("base-sync");
5271        let rebased =
5272            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5273                .await
5274            {
5275                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5276                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5277                Err(e) => Err(e),
5278            };
5279        let attempts = attempts + 1;
5280        let stop_kind = match &rebased {
5281            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5282            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5283            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5284        };
5285        match rebased {
5286            Ok(None) => {
5287                // The branch ref moved, but a worktree that already had it
5288                // checked out (the winner's) was not told; sync its index and
5289                // files before anything reads them.
5290                git::sync_to_head(&winner.worktree).await?;
5291                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5292                let mut conflict = None;
5293                if let Some(pinned) = &remote_tip {
5294                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5295                    match pushed {
5296                        Ok(o) if o.ok() => self.state.event(
5297                            "land",
5298                            format!("pushed rebased {} to {remote}", winner.branch),
5299                        ),
5300                        Ok(o) => {
5301                            conflict = Some(format!(
5302                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5303                                winner.branch,
5304                                short(pinned),
5305                                o.stderr.chars().take(600).collect::<String>()
5306                            ));
5307                        }
5308                        Err(e) => {
5309                            conflict = Some(format!(
5310                                "rebased {} locally but could not push it: {e:#}",
5311                                winner.branch
5312                            ));
5313                        }
5314                    }
5315                }
5316                if let Some(why) = &conflict {
5317                    self.state.status = RunStatus::Blocked;
5318                    self.state.event("land", why.clone());
5319                }
5320                self.state.base_sync = Some(BaseSync {
5321                    tip: tip.clone(),
5322                    behind: 0,
5323                    attempts,
5324                    conflict,
5325                    already_in: None,
5326                });
5327                self.state
5328                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
5329            }
5330            Ok(Some(conflict)) => {
5331                let why = format!(
5332                    "{} conflicts with {tracking} and did not rebase: {}",
5333                    winner.branch,
5334                    conflict.chars().take(600).collect::<String>()
5335                );
5336                self.state.status = RunStatus::Blocked;
5337                self.state.base_sync = Some(BaseSync {
5338                    tip,
5339                    behind,
5340                    attempts,
5341                    conflict: Some(why.clone()),
5342                    already_in: None,
5343                });
5344                self.state.event("land", why);
5345            }
5346            // A refused Git write or a busy repository: nothing is wrong with
5347            // the branch. The scratch rebase stays standing and the run
5348            // resumes into it (`base_sync` must not claim a conflict).
5349            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5350                self.state.save()?;
5351                return Ok(());
5352            }
5353            Err(e) => {
5354                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5355                self.state.status = RunStatus::Blocked;
5356                self.state.base_sync = Some(BaseSync {
5357                    tip,
5358                    behind,
5359                    attempts,
5360                    conflict: Some(why.clone()),
5361                    already_in: None,
5362                });
5363                self.state.event("land", why);
5364            }
5365        }
5366        if !self.state.base_sync_blocked() {
5367            self.state.base_sync_recovery.stop = None;
5368        }
5369        self.hold_base_sync_kind(stop_kind).await?;
5370        self.state.save()?;
5371        Ok(())
5372    }
5373
5374    /// Push a branch an earlier attempt rebased locally but never published,
5375    /// pinned to the remote tip read right after a successful fetch. Returns
5376    /// the reason when the run must stop; `None` when there was nothing to do
5377    /// (no remote copy, the same tip, or a remote copy this branch already
5378    /// contains, which the PR-time push fast-forwards) or the push succeeded.
5379    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5380        // Task competitions publish only after the PR text gate, to their
5381        // public head; never resurrect a candidate branch on the remote.
5382        if self.state.task_pr_branch().is_some() {
5383            return None;
5384        }
5385        let repo = self.state.repo.clone();
5386        let remote = self.state.config.merge.remote.clone();
5387        let fetched = git::fetch(&repo, &remote, branch).await;
5388        if !matches!(&fetched, Ok(o) if o.ok()) {
5389            return None;
5390        }
5391        let branch_tracking = format!("{remote}/{branch}");
5392        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5393        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5394            return None;
5395        }
5396        if !crate::reconcile::origin_missing(&repo, head, &theirs)
5397            .await
5398            .is_ok_and(|missing| missing.is_empty())
5399        {
5400            return Some(format!(
5401                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5402                 them",
5403                short(&theirs)
5404            ));
5405        }
5406        match git::push_pinned(&repo, &remote, branch, &theirs).await {
5407            Ok(o) if o.ok() => {
5408                self.state
5409                    .event("land", format!("pushed rebased {branch} to {remote}"));
5410                None
5411            }
5412            Ok(o) => Some(format!(
5413                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5414                 someone may have pushed): {}",
5415                short(&theirs),
5416                o.stderr.chars().take(600).collect::<String>()
5417            )),
5418            Err(e) => Some(format!(
5419                "{branch} is rebased locally but could not be pushed: {e:#}"
5420            )),
5421        }
5422    }
5423
5424    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
5425    /// is already on `tip` under other commit ids ([`crate::already`]); returns
5426    /// whether it did.
5427    ///
5428    /// Checked only when the base is ahead of the branch. A failing check is
5429    /// "not proven" - the ordinary rebase path then decides - never a reason to
5430    /// stop the run.
5431    ///
5432    /// The remote copy of the branch is held to the same standard as the local
5433    /// one: if it carries a tip this worktree does not, that tip must itself be
5434    /// proven in the base, or nothing is settled (a pull request would
5435    /// otherwise be closed over commits nobody checked). The pull request is
5436    /// closed *before* the terminal status is saved; if that fails for a
5437    /// reason other than a refusal (no network, a `gh` error) the run is left
5438    /// `Blocked` with the reason as its conflict, which a resume retries -
5439    /// the same recovery a phantom conflict gets.
5440    async fn settle_already_in(
5441        &mut self,
5442        branch: &str,
5443        tip: &str,
5444        head: &str,
5445        attempts: usize,
5446        behind: usize,
5447    ) -> Result<bool> {
5448        let repo = self.state.repo.clone();
5449        let remote = self.state.config.merge.remote.clone();
5450        let start = self.state.base_commit.clone();
5451        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5452            Ok(Some(e)) => e,
5453            Ok(None) => return Ok(false),
5454            Err(e) => {
5455                tracing::warn!("already-in-base check for {branch}: {e:#}");
5456                return Ok(false);
5457            }
5458        };
5459        let mut verified = vec![head.to_owned()];
5460        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5461        let fetched = git::fetch(&repo, &remote, &public_branch).await;
5462        if matches!(&fetched, Ok(o) if o.ok())
5463            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5464            && theirs != head
5465        {
5466            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5467                Ok(Some(_)) => verified.push(theirs),
5468                _ => return Ok(false),
5469            }
5470        }
5471        let base_branch = self.state.base_branch.clone();
5472        let message = format!(
5473            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5474             land",
5475            evidence.names(),
5476            evidence.proof.as_str()
5477        );
5478        let closed =
5479            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5480                .await;
5481        match closed {
5482            Ok(Ok(url)) => self
5483                .state
5484                .event("land", format!("closed {url}: superseded on {base_branch}")),
5485            Ok(Err(why)) => self
5486                .state
5487                .event("land", format!("did not close a pull request: {why}")),
5488            Err(e) => {
5489                let why = format!(
5490                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
5491                     not be closed ({e:#}); resume to retry"
5492                );
5493                self.state.status = RunStatus::Blocked;
5494                self.state.base_sync = Some(BaseSync {
5495                    tip: tip.to_owned(),
5496                    behind,
5497                    attempts,
5498                    conflict: Some(why.clone()),
5499                    already_in: None,
5500                });
5501                self.state.event("land", why);
5502                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5503                    .await?;
5504                self.state.save()?;
5505                return Ok(true);
5506            }
5507        }
5508        self.state.status = RunStatus::AlreadyInBase;
5509        self.state.base_sync_recovery.stop = None;
5510        self.state.base_sync = Some(BaseSync {
5511            tip: tip.to_owned(),
5512            behind,
5513            attempts,
5514            conflict: None,
5515            already_in: Some(evidence),
5516        });
5517        self.state.event("land", message);
5518        self.state.save()?;
5519        self.settle_questions();
5520        Ok(true)
5521    }
5522
5523    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
5524    /// last landed the winner on, once it has run, else the commit the run
5525    /// branched from.
5526    ///
5527    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
5528    /// and `vote` all happen before there is a winner to rebase, so they
5529    /// compare every candidate against the branch point on purpose, and a
5530    /// base that moves after they are already done cannot change an answer
5531    /// they already gave.
5532    fn landing_base(&self) -> String {
5533        self.state
5534            .base_sync
5535            .as_ref()
5536            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5537    }
5538
5539    // ------------------------------------------------------- operator fix
5540
5541    /// Route specific, already-recorded review findings to a fixer for a
5542    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5543    /// entry point.
5544    ///
5545    /// Distinct from `review_loop`'s own fix step in three ways: it never
5546    /// runs a reviewer wave, it never spends review-round budget, and what
5547    /// happened is recorded as an [`OperatorFixRequest`] appended to
5548    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5549    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5550    /// and vote must never be rewritten to look like a manufactured blocking
5551    /// verdict.
5552    ///
5553    /// Only meaningful once review has actually concluded: `Ready` (handed
5554    /// off with findings still open, or simply concluded clean while minor
5555    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5556    /// gate failed). Everything else is refused: a run still in progress
5557    /// should simply be resumed, and a `Merged` run's branch has already
5558    /// landed — reopening *this* run's own record cannot change that, so the
5559    /// answer there is a fresh `magi review <branch>`.
5560    ///
5561    /// A real commit here re-verifies through a fresh, ordinary review-only
5562    /// run on the same branch ([`Self::review`]) rather than reopening this
5563    /// run's own `review_loop`: once any round in this run's history went
5564    /// clean, `review_conclusion` treats that as permanent by design (the
5565    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5566    /// way to force one more genuine reviewer wave out of *this* run without
5567    /// either rewriting history or weakening that guarantee for every other
5568    /// caller. A review-only run costs nothing extra — no implementation, no
5569    /// judging, no vote — and exercises the exact same review → verify →
5570    /// gate → (human) merge path, unmodified.
5571    pub async fn fix_selected(
5572        &mut self,
5573        ids: &[String],
5574        reason: &str,
5575        allow_stale: bool,
5576    ) -> Result<()> {
5577        let reason = reason.trim();
5578        if reason.is_empty() {
5579            bail!("a fix request needs a reason — that is the operator's own record of why");
5580        }
5581        if ids.is_empty() {
5582            bail!("no finding id given");
5583        }
5584        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5585            bail!(
5586                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5587                 has already concluded — can be given a targeted fix. A run still \
5588                 in progress should simply be resumed; a `merged` run's branch has \
5589                 already landed, so its answer is a fresh `magi review <branch>`, \
5590                 not reopening this run's own record",
5591                self.state.id,
5592                self.state.status.as_str()
5593            );
5594        }
5595        let Some(winner) = self.state.winner().cloned() else {
5596            bail!("run {} has no winning candidate to fix", self.state.id);
5597        };
5598        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5599            bail!(
5600                "branch `{}` no longer exists; this run cannot be extended",
5601                winner.branch
5602            );
5603        }
5604        let home = crate::run::home();
5605        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5606            bail!(
5607                "run {} is currently being worked on by another magi process",
5608                self.state.id
5609            );
5610        }
5611        // Held for the rest of this call, including the follow-up review
5612        // below: two `magi fix` invocations against the same run must not
5613        // both reach the worktree manipulation further down, which would
5614        // otherwise race to remove and recreate the same directory — see
5615        // [`FixClaim`]'s own doc.
5616        let _claim = FixClaim::acquire(&self.state.dir())?;
5617
5618        // Resolve every id before spending anything — an unknown id refuses
5619        // the whole request rather than silently dropping it — and dedup
5620        // while keeping the operator's own order.
5621        let mut seen = BTreeSet::new();
5622        let mut findings = Vec::new();
5623        let mut missing = Vec::new();
5624        for id in ids {
5625            if !seen.insert(id.clone()) {
5626                continue;
5627            }
5628            match self.state.finding(id) {
5629                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5630                    id: f.id.clone(),
5631                    severity: f.severity,
5632                    reviewer_vote: rec.vote,
5633                    round: round.round,
5634                    round_head: round.head.clone(),
5635                    reviewer: rec.reviewer,
5636                    agent: rec.agent.clone(),
5637                    file: f.file.clone(),
5638                    line: f.line,
5639                    title: f.title.clone(),
5640                    detail: f.detail.clone(),
5641                    outcome: OperatorFixOutcome::Pending,
5642                }),
5643                None => missing.push(id.clone()),
5644            }
5645        }
5646        if !missing.is_empty() {
5647            bail!(
5648                "unknown finding id(s): {}; nothing was changed",
5649                missing.join(", ")
5650            );
5651        }
5652
5653        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5654        let stale_details: Vec<(String, String)> = findings
5655            .iter()
5656            .filter(|f| f.round_head != head_at_request)
5657            .map(|f| (f.id.clone(), f.round_head.clone()))
5658            .collect();
5659        let stale = !stale_details.is_empty();
5660        if stale && !allow_stale {
5661            bail!(
5662                "the branch has moved since some finding(s) were raised — {} — now \
5663                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5664                stale_details
5665                    .iter()
5666                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5667                    .collect::<Vec<_>>()
5668                    .join(", "),
5669                short(&head_at_request)
5670            );
5671        }
5672
5673        let request = OperatorFixRequest {
5674            requested_at: Timestamp::now(),
5675            reason: reason.to_owned(),
5676            findings,
5677            head_at_request: head_at_request.clone(),
5678            allow_stale,
5679            stale,
5680            fix: None,
5681            result_head: None,
5682            follow_up_review_run: None,
5683        };
5684        self.state.event(
5685            "fix",
5686            format!(
5687                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5688                request.findings.len(),
5689                request
5690                    .findings
5691                    .iter()
5692                    .map(|f| f.id.as_str())
5693                    .collect::<Vec<_>>()
5694                    .join(", "),
5695            ),
5696        );
5697        // Recorded now, before any worktree work or the fixer call itself —
5698        // and re-saved at each checkpoint below: a crash at any point after
5699        // this (mid fixer call, mid follow-up review) must not lose the fact
5700        // that this was requested, for which findings, and why. Everything
5701        // past this point reads and writes through `request_index` rather
5702        // than a local variable, since `request` itself is moved here.
5703        self.state.operator_fixes.push(request);
5704        self.state.save()?;
5705        let request_index = self.state.operator_fixes.len() - 1;
5706
5707        // A fresh, dedicated worktree for this one call, never the winner's
5708        // own worktree in place: that one may already be gone (folded away),
5709        // and reusing it in place would leave the branch checked out there
5710        // when the follow-up review below tries to check it out again. Freed
5711        // immediately after, either way — but only once confirmed clean:
5712        // `worktree_remove` is a `git worktree remove --force`, which would
5713        // otherwise discard uncommitted work left there by the operator or
5714        // another process before this had a chance to even look at it.
5715        let repo = self.state.repo.clone();
5716        let coordination = self.coordinate(&repo).await?;
5717        if winner.worktree.exists() {
5718            // Lockfiles a rescue commit withheld stay untracked on purpose and
5719            // are already recorded; they are not the operator's work to protect.
5720            let dirty = git::git(
5721                &winner.worktree,
5722                &["status", "--porcelain", "--untracked-files=all"],
5723            )
5724            .await?;
5725            let only_withheld = dirty.lines().all(|l| {
5726                l.strip_prefix("?? ")
5727                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5728            });
5729            if !only_withheld {
5730                bail!(
5731                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5732                     discard them first",
5733                    winner.worktree.display()
5734                );
5735            }
5736            git::worktree_remove(&self.state.repo, &winner.worktree)
5737                .await
5738                .ok();
5739        }
5740        let fix_worktree = self.state.worktree_root().join("operator-fix");
5741        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5742        git::git(
5743            &self.state.repo,
5744            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5745        )
5746        .await
5747        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5748        if !git::is_clean(&fix_worktree).await? {
5749            git::worktree_remove(&self.state.repo, &fix_worktree)
5750                .await
5751                .ok();
5752            bail!(
5753                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5754                winner.branch
5755            );
5756        }
5757        let repo_for_setup = self.state.repo.clone();
5758        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5759            Ok(()) => {
5760                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5761            }
5762            Err(e) => Err(e),
5763        };
5764        if let Err(e) = setup {
5765            git::worktree_remove(&self.state.repo, &fix_worktree)
5766                .await
5767                .ok();
5768            return Err(e);
5769        }
5770
5771        // This explicit operator request retries availability independently
5772        // of execute(); the previous call has already saved and cleaned up.
5773        self.state.resource_block = None;
5774
5775        let run_id = self.state.id.clone();
5776        let prompts = self.state.config.prompts.clone();
5777        let language = self.state.config.graph.language.clone();
5778        let sessions = self.state.config.graph.sessions;
5779        let artifacts = agent::artifacts_dir(&self.state.dir());
5780        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5781            .findings
5782            .iter()
5783            .map(|f| Finding {
5784                id: f.id.clone(),
5785                severity: f.severity,
5786                file: f.file.clone(),
5787                line: f.line,
5788                title: f.title.clone(),
5789                detail: f.detail.clone(),
5790            })
5791            .collect();
5792        let fix_prompt = prompt::operator_fix_in(
5793            &self.state.instruction,
5794            &finding_list,
5795            reason,
5796            &stale_details,
5797            &head_at_request,
5798            &language,
5799            &self.state.config.graph.github_language,
5800        );
5801        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5802        drop(coordination);
5803        let (job, seat, out) = self
5804            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5805                prompt: fix_prompt.clone(),
5806                spec,
5807                seat,
5808                cwd: fix_worktree.clone(),
5809                timeout,
5810                allow_write: true,
5811                sessions,
5812                artifacts: artifacts.clone(),
5813                stem: "operator-fix".to_owned(),
5814                handover: None,
5815            })
5816            .await;
5817        let agent_id = seat.agent.clone();
5818
5819        let mut fix = FixRecord {
5820            agent: agent_id,
5821            addressed: Vec::new(),
5822            rejected: Vec::new(),
5823            notes: String::new(),
5824            committed: false,
5825            failed: None,
5826            duration_ms: 0,
5827            continuation: None,
5828        };
5829        let mut final_seat = seat.clone();
5830        match out {
5831            AgentOutcome::Ok(o) => {
5832                fix.duration_ms = o.duration_ms;
5833                let parsed = verdict::extract_json::<FixReport>(&o.text);
5834                let incomplete_reason = match &parsed {
5835                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5836                        "the reply parsed, but it reported a command whose own CLI \
5837                         never confirmed an exit status"
5838                            .to_owned(),
5839                    ),
5840                    Ok(_) => None,
5841                    Err(e) => Some(e.to_string()),
5842                };
5843                match incomplete_reason {
5844                    None => {
5845                        let report = parsed.expect("checked Ok above");
5846                        fix.addressed = report.addressed;
5847                        fix.rejected = report.rejected;
5848                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5849                    }
5850                    Some(reason) => {
5851                        let (resumed_seat, resolved, failure, cont) = self
5852                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5853                            .await;
5854                        fix.duration_ms += cont.cumulative_wait_ms;
5855                        fix.continuation = Some(cont);
5856                        final_seat = resumed_seat;
5857                        match resolved {
5858                            Some(report) => {
5859                                fix.addressed = report.addressed;
5860                                fix.rejected = report.rejected;
5861                                fix.notes =
5862                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5863                            }
5864                            None => fix.failed = failure,
5865                        }
5866                    }
5867                }
5868            }
5869            AgentOutcome::Dropped(o) => {
5870                fix.duration_ms = o.duration_ms;
5871                let why = o
5872                    .dropped
5873                    .as_ref()
5874                    .map(|d| d.why.as_str())
5875                    .unwrap_or("the CLI ended the stream without delivering its answer");
5876                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5877            }
5878            AgentOutcome::Quota(o) => {
5879                self.state.quota.push(QuotaLoss::from_report(
5880                    final_seat.key.clone(),
5881                    "fix",
5882                    o.quota.as_ref(),
5883                ));
5884                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5885            }
5886            AgentOutcome::Failed(e) => fix.failed = Some(e),
5887            AgentOutcome::ResourceBlocked(e) => {
5888                fix.failed = Some(format!("the fixer was not started: {e}"));
5889            }
5890        }
5891        // Even a deferred call must rescue edits, release the worktree and
5892        // finalize its record below so an explicit `magi fix` can retry.
5893        if fix.continuation.is_none() {
5894            fix.continuation = Some(ContinuationRecord::not_needed());
5895        }
5896        self.state.seats.insert(final_seat.key.clone(), final_seat);
5897
5898        let rescue_message = format!(
5899            "magi: operator-selected fix ({}) (uncommitted work)",
5900            self.state.operator_fixes[request_index]
5901                .findings
5902                .iter()
5903                .map(|f| f.id.as_str())
5904                .collect::<Vec<_>>()
5905                .join(", ")
5906        );
5907        let repo = self.state.repo.clone();
5908        let coordination = self.coordinate(&repo).await?;
5909        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5910            self.state.note_withheld("fix", &r.withheld);
5911        }
5912        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5913        fix.committed = after != head_at_request;
5914        git::worktree_remove(&self.state.repo, &fix_worktree)
5915            .await
5916            .ok();
5917        drop(coordination);
5918
5919        self.state.event(
5920            "fix",
5921            match &fix.failed {
5922                Some(reason) => format!(
5923                    "operator fix: adoption report was lost ({reason}); {}",
5924                    if fix.committed {
5925                        "committed"
5926                    } else {
5927                        "NO new commit"
5928                    }
5929                ),
5930                None => format!(
5931                    "operator fix: {} addressed, {} rejected, {}",
5932                    fix.addressed.len(),
5933                    fix.rejected.len(),
5934                    if fix.committed {
5935                        "committed"
5936                    } else {
5937                        "NO new commit"
5938                    }
5939                ),
5940            },
5941        );
5942
5943        // Every selected finding gets an outcome — never left `Pending` once
5944        // the fixer's own turn is over. A report that never came back at all
5945        // marks every one of them `Unreported`, not silently "not addressed":
5946        // quota, a dropped stream, or an exhausted continuation are gaps in
5947        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5948        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5949        for f in &mut self.state.operator_fixes[request_index].findings {
5950            f.outcome = if fix.failed.is_some() {
5951                OperatorFixOutcome::Unreported
5952            } else if fix.addressed.contains(&f.id) {
5953                OperatorFixOutcome::Addressed
5954            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5955                OperatorFixOutcome::Rejected { why: r.why.clone() }
5956            } else {
5957                OperatorFixOutcome::Unreported
5958            };
5959        }
5960
5961        let committed = fix.committed;
5962        if committed {
5963            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5964        }
5965        self.state.operator_fixes[request_index].fix = Some(fix);
5966        // Saved again now that the fixer's own outcome is final, on top of
5967        // the save right after the request was first pushed above.
5968        self.state.save()?;
5969        if self.state.resource_block.is_some() {
5970            return self.stop_for_resource();
5971        }
5972
5973        if committed {
5974            self.state.event(
5975                "fix",
5976                format!(
5977                    "operator fix committed {}; opening a follow-up review-only run",
5978                    short(&after)
5979                ),
5980            );
5981            // The operator asked for the fix, and the follow-up serves whatever
5982            // task the run it follows served.
5983            let origin =
5984                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5985            match Self::review(
5986                &self.state.repo,
5987                &winner.branch,
5988                self.state.config.clone(),
5989                origin,
5990            )
5991            .await
5992            {
5993                Ok(mut follow_up) => {
5994                    follow_up.state.event(
5995                        "start",
5996                        format!(
5997                            "requested by an operator fix on run {} for finding(s) {}",
5998                            self.state.id,
5999                            self.state.operator_fixes[request_index]
6000                                .findings
6001                                .iter()
6002                                .map(|f| f.id.as_str())
6003                                .collect::<Vec<_>>()
6004                                .join(", "),
6005                        ),
6006                    );
6007                    follow_up.state.save()?;
6008                    let follow_up_id = follow_up.state.id.clone();
6009                    // The follow-up is a run like any other: it belongs to the
6010                    // task of the run it follows, or to one filed for it.
6011                    let adopted = match crate::direct::adopt(
6012                        &follow_up.state,
6013                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6014                    ) {
6015                        Ok(a) => a,
6016                        Err(e) => {
6017                            // An ownerless run must not spend agent calls; it
6018                            // stays saved, and `magi run --resume` adopts it.
6019                            self.state.event(
6020                                "fix",
6021                                format!(
6022                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6023                                ),
6024                            );
6025                            self.state.save()?;
6026                            return Ok(());
6027                        }
6028                    };
6029                    let executed = follow_up.execute().await;
6030                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6031                    if let Some(a) = adopted {
6032                        a.finish(&follow_up.state, executed);
6033                    }
6034                    if let Some(e) = failure {
6035                        self.state.event(
6036                            "fix",
6037                            format!(
6038                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6039                            ),
6040                        );
6041                    }
6042                    self.state.operator_fixes[request_index].follow_up_review_run =
6043                        Some(follow_up_id);
6044                }
6045                Err(e) => {
6046                    self.state.event(
6047                        "fix",
6048                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6049                    );
6050                }
6051            }
6052            self.state.save()?;
6053        }
6054
6055        Ok(())
6056    }
6057
6058    // --------------------------------------------------------------- review
6059
6060    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6061    /// names, in order, each at most once, moving on only when the call
6062    /// advances (an error, a quota hit or nothing usable - the same decision
6063    /// point `agent::chain_advances` is for the other chained roles).
6064    ///
6065    /// Each agent gets its own seat from `Runner::seat`: the same agent
6066    /// continues its conversation (the winner's own implementer seat when it
6067    /// is the winner's author, now that the competition is over), another
6068    /// takes a fresh one so the full prompt is sent again. A handover is
6069    /// recorded under `node`, which is what makes the fallback stick for the
6070    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6071    /// is returned, so an exhausted chain reads exactly like a single failed
6072    /// fixer: one quota loss, the same wording, the same refund. An earlier
6073    /// attempt's edits are left in the tree and judged, with the final
6074    /// attempt's, by what git says afterwards.
6075    ///
6076    /// Returns the job that produced the outcome, for `continue_fix_report`.
6077    async fn ask_fixer(
6078        &mut self,
6079        winner: &Candidate,
6080        node: &'static str,
6081        round: Option<usize>,
6082        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6083    ) -> (SeatJob, SeatState, AgentOutcome) {
6084        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6085        self.ask_fixer_among(attempts, node, round, None, build)
6086            .await
6087    }
6088
6089    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6090    /// worktree and a commit: before the next agent is asked after one that
6091    /// did not satisfy the call, the worktree is put back on that commit, so
6092    /// a failed agent's partial edits are not carried into the next one.
6093    async fn ask_fixer_among(
6094        &mut self,
6095        attempts: Vec<(AgentSpec, String)>,
6096        node: &'static str,
6097        round: Option<usize>,
6098        restore: Option<(&Path, &str)>,
6099        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6100    ) -> (SeatJob, SeatState, AgentOutcome) {
6101        let run_id = self.state.id.clone();
6102        let prompts = self.state.config.prompts.clone();
6103        let cache = self.state.config.cache_dir();
6104        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6105        let mut last = None;
6106        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6107            let seat = self.seat(&key, &spec.id);
6108            let mut job = build(spec.clone(), seat);
6109            if i > 0 {
6110                job.stem = format!("{}-{}", job.stem, spec.id);
6111            }
6112            let ctx = WaveCtx {
6113                carry_seats: false,
6114                pauses: Some((&self.pause, &self.interrupt)),
6115                run: &run_id,
6116                node,
6117                prompts: &prompts,
6118                cache: cache.as_deref(),
6119                round,
6120            };
6121            let (seat, out) =
6122                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6123            // A resource block never advances the chain: the chosen agent
6124            // was not asked, so it must be the one asked again on resume.
6125            let advances = match &out {
6126                AgentOutcome::Ok(o) => agent::output_advances(o),
6127                AgentOutcome::ResourceBlocked(_) => false,
6128                _ => true,
6129            };
6130            if let Some(next) = ids.get(i + 1)
6131                && advances
6132            {
6133                self.state.seats.insert(seat.key.clone(), seat.clone());
6134                let class =
6135                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6136                let reason = match &out {
6137                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6138                    other => fail_reason(other),
6139                };
6140                // A failed agent's edits must not reach the next one, and a
6141                // restore that did not work ends the chain: the call fails
6142                // and the caller's own (confirmed) restore decides.
6143                if let Some((wt, rev)) = restore
6144                    && let Err(e) = self.graft_restore(wt, rev).await
6145                {
6146                    let why =
6147                        format!("could not restore the worktree before the next fixer: {e:#}");
6148                    tracing::warn!("{why}");
6149                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6150                        AgentOutcome::ResourceBlocked(why)
6151                    } else {
6152                        AgentOutcome::Failed(why)
6153                    };
6154                    last = Some((job, seat, out));
6155                    break;
6156                }
6157                record_handover(
6158                    &mut self.state,
6159                    node,
6160                    &seat.key,
6161                    &spec.id,
6162                    next,
6163                    &class,
6164                    &reason,
6165                );
6166                continue;
6167            }
6168            last = Some((job, seat, out));
6169            break;
6170        }
6171        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6172        if let AgentOutcome::Quota(output) = &out
6173            && node != "graft"
6174        {
6175            // Graft is best effort: its caller records the loss and restores
6176            // the pre-graft tree, then proceeds to review and gate.
6177            // Every fallback was tried. Machine outages cannot be repaired by
6178            // spending a fix round; use each caller's resumable resource path.
6179            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6180            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6181            self.state.quota.push(loss);
6182            self.state.seats.insert(seat.key.clone(), seat.clone());
6183            self.state.status = RunStatus::Blocked;
6184            return (
6185                job,
6186                seat,
6187                AgentOutcome::ResourceBlocked(
6188                    "fixer unavailable; restore agent availability and resume".into(),
6189                ),
6190            );
6191        }
6192        (job, seat, out)
6193    }
6194
6195    /// The one repair a verification-only failure earns (see
6196    /// [`crate::recovery`]), before the first review round of a run that took
6197    /// a branch over.
6198    ///
6199    /// Saved as started *before* the fixer is called, so a resume never
6200    /// repairs twice; a start found on re-entry is rolled back to the commit
6201    /// it began from. A failed or unusable fixer leaves the tree as it was and
6202    /// the review goes on over it, so the first round's own verification
6203    /// reports the failure again. The failure is never carried into a round as
6204    /// a result: verification is re-run on whatever commit this leaves.
6205    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6206        use crate::recovery::{Mode, Repair};
6207        if !self.state.reviews.is_empty() {
6208            return Ok(());
6209        }
6210        let Some(mut rec) = self.state.recovery.clone() else {
6211            return Ok(());
6212        };
6213        if rec.mode != Mode::VerifyRepair {
6214            return Ok(());
6215        }
6216        let wt = winner.worktree.clone();
6217        let mut head = git::rev_parse(&wt, "HEAD").await?;
6218        match rec.repair.clone() {
6219            Repair::Done { .. } => return Ok(()),
6220            Repair::WaitingForRestore { before } => {
6221                self.graft_restore(&wt, &before).await?;
6222                head = before;
6223                rec.repair = Repair::NotStarted;
6224                self.state.recovery = Some(rec.clone());
6225                self.state.save()?;
6226            }
6227            Repair::Started { before } => {
6228                // Interrupted: the unfinished edits are not trusted.
6229                // An unconfirmed rollback blocks the run with `Started` kept,
6230                // so a resume retries it; the tree is never reviewed as is.
6231                if let Err(e) = self.graft_restore(&wt, &before).await {
6232                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6233                        return Err(e);
6234                    }
6235                    return self
6236                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6237                        .await;
6238                }
6239                rec.repair = Repair::Done {
6240                    before: before.clone(),
6241                    after: before,
6242                    committed: false,
6243                    failed: Some("interrupted; rolled back".to_owned()),
6244                };
6245                self.state
6246                    .event("recovery", "the repair was interrupted and rolled back");
6247                self.state.recovery = Some(rec);
6248                self.state.save()?;
6249                return Ok(());
6250            }
6251            Repair::NotStarted => {}
6252        }
6253        if crate::recovery::recheck(&mut rec, &head) {
6254            self.state.event("recovery", rec.summary());
6255            self.state.recovery = Some(rec);
6256            self.state.save()?;
6257            return Ok(());
6258        }
6259        rec.repair = Repair::Started {
6260            before: head.clone(),
6261        };
6262        self.state.recovery = Some(rec.clone());
6263        self.state.save()?;
6264
6265        let language = self.state.config.graph.language.clone();
6266        let sessions = self.state.config.graph.sessions;
6267        let artifacts = agent::artifacts_dir(&self.state.dir());
6268        let fix_prompt = prompt::recovery_fix(
6269            &self.state.instruction,
6270            &rec,
6271            self.state.config.graph.review_rounds,
6272            &language,
6273        );
6274        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6275        self.state.event(
6276            "recovery",
6277            format!(
6278                "repair: the fixer starts from the failed verification on {}",
6279                short(&head)
6280            ),
6281        );
6282        let (_job, seat, out) = self
6283            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6284                prompt: fix_prompt.clone(),
6285                spec,
6286                seat,
6287                cwd: wt.clone(),
6288                timeout,
6289                allow_write: true,
6290                sessions,
6291                artifacts: artifacts.clone(),
6292                stem: "fix-recovery".to_owned(),
6293                handover: None,
6294            })
6295            .await;
6296        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6297            rec.repair = Repair::WaitingForRestore { before: head };
6298            self.state.recovery = Some(rec);
6299            return self.stop_for_resource();
6300        }
6301        let failed = match &out {
6302            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6303            AgentOutcome::Quota(o) => {
6304                self.state.quota.push(QuotaLoss::from_report(
6305                    seat.key.clone(),
6306                    "fix",
6307                    o.quota.as_ref(),
6308                ));
6309                Some("rate limited (quota); the fixer could not run".to_owned())
6310            }
6311            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6312            AgentOutcome::Failed(e) => Some(e.clone()),
6313        };
6314        self.state.seats.insert(seat.key.clone(), seat);
6315        let coordination = self.coordinate(&wt).await?;
6316        let failed = match failed {
6317            // The agent's partial edits are not kept.
6318            Some(why) => {
6319                if let Err(e) = git::restore_to(&wt, &head).await {
6320                    return self
6321                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6322                        .await;
6323                }
6324                Some(why)
6325            }
6326            None => {
6327                if let Ok(r) = git::rescue_commit(
6328                    &wt,
6329                    "magi: recovery repair of the failed verification (uncommitted work)",
6330                )
6331                .await
6332                {
6333                    self.state.note_withheld("fix", &r.withheld);
6334                }
6335                None
6336            }
6337        };
6338        let after = git::rev_parse(&wt, "HEAD").await?;
6339        drop(coordination);
6340        self.state.event(
6341            "recovery",
6342            match &failed {
6343                Some(why) => format!("repair: nothing kept ({why})"),
6344                None if after != head => {
6345                    format!("repair: committed {}", short(&after))
6346                }
6347                None => "repair: the fixer made no commit".to_owned(),
6348            },
6349        );
6350        rec.repair = Repair::Done {
6351            before: head.clone(),
6352            committed: after != head,
6353            after,
6354            failed,
6355        };
6356        self.state.recovery = Some(rec);
6357        self.state.save()?;
6358        Ok(())
6359    }
6360
6361    /// A repair whose rollback could not be confirmed: the run is blocked and
6362    /// resumable, and the unconfirmed state is never reviewed.
6363    async fn stop_recovery(
6364        &mut self,
6365        mut rec: crate::recovery::RecoveryRecord,
6366        head: &str,
6367        why: String,
6368    ) -> Result<()> {
6369        // `Started` stays recorded: a resume retries the restore.
6370        rec.repair = crate::recovery::Repair::Started {
6371            before: head.to_owned(),
6372        };
6373        self.state.recovery = Some(rec);
6374        self.state
6375            .event("recovery", format!("repair could not be undone: {why}"));
6376        self.state.status = RunStatus::Blocked;
6377        self.state.save()?;
6378        bail!("recovery repair could not be rolled back: {why}")
6379    }
6380
6381    async fn review_loop(&mut self) -> Result<()> {
6382        if self.hold_base_sync().await? {
6383            return Ok(());
6384        }
6385        loop {
6386            if self
6387                .state
6388                .base_sync
6389                .as_ref()
6390                .is_some_and(|s| s.conflict.is_some())
6391            {
6392                return Ok(());
6393            }
6394            if self.catchup_repairable() {
6395                let record = self.state.reviews.last().expect("repairable round").clone();
6396                let winner = self
6397                    .state
6398                    .winner()
6399                    .cloned()
6400                    .context("catch-up repair has no winner")?;
6401                if !self.fix_review_round(&winner, record).await? {
6402                    return Ok(());
6403                }
6404            }
6405            self.review_loop_pass().await?;
6406            if !self.catchup_repairable() {
6407                return Ok(());
6408            }
6409        }
6410    }
6411
6412    /// Derive an owed catch-up repair from durable round evidence, including
6413    /// reentry after a crash before the fixer was dispatched.
6414    fn catchup_repairable(&self) -> bool {
6415        self.state.reviews.len() < self.state.config.graph.review_rounds
6416            && self.state.reviews.last().is_some_and(|r| {
6417                !r.clean
6418                    && r.blocking == 0
6419                    && r.fix.is_none()
6420                    && r.e2e_status() == E2eStatus::Failed
6421            })
6422    }
6423
6424    async fn review_loop_pass(&mut self) -> Result<()> {
6425        // A base that would not rebase is a person's decision, not a review
6426        // round: nothing here would change the answer, and reviewers and a
6427        // fixer would be spending real budget on a tree that cannot land
6428        // regardless of what they find.
6429        if self
6430            .state
6431            .base_sync
6432            .as_ref()
6433            .is_some_and(|s| s.conflict.is_some())
6434        {
6435            return Ok(());
6436        }
6437        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
6438        // agent files with `magi task add` name the run that paid for it. The
6439        // prompt overlay is cloned alongside it because the waves borrow it
6440        // while `self` is mutably borrowed by the node's own bookkeeping.
6441        let run_id = self.state.id.clone();
6442        let prompts = self.state.config.prompts.clone();
6443        let Some(winner) = self.state.winner().cloned() else {
6444            return Ok(());
6445        };
6446        let max_rounds = self.state.config.graph.review_rounds;
6447        // A clean round, an exhausted round budget, or a stalled tree (see
6448        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
6449        // they are recorded — recomputed here, not read off `status`, so a
6450        // reentry into a run that already stopped restates the identical
6451        // verdict instead of silently handing back whatever an earlier node
6452        // in this same walk clobbered `status` to (a solo-candidate
6453        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
6454        // below runs an empty range once the budget is spent, and would
6455        // otherwise fall through without touching `status` at all.
6456        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
6457            // A reentry after a crash between the last round's save and
6458            // `stop_reviewing` reaches the hand-off here, not there.
6459            if status == RunStatus::Gating {
6460                self.record_contested_handoff();
6461            }
6462            self.state.status = status;
6463            self.state.save()?;
6464            return Ok(());
6465        }
6466        self.state.status = RunStatus::Reviewing;
6467        // A last recorded round whose own verification never resolved
6468        // (`ResourceBlocked` — the shared build cache, not the patch) is
6469        // never a concluded round, whatever the round budget says: starting
6470        // a fresh round on top of it would spend a whole new reviewer wave
6471        // re-reading an unchanged patch instead of just retrying the one
6472        // check that actually needs it, and once the budget is spent the
6473        // loop below has nothing left to do at all (its range is empty).
6474        // Retry that check directly instead, exactly the same retry
6475        // `stop_reviewing` already does for its own catch-up case.
6476        if self.state.reviews.last().is_some_and(|r| {
6477            r.e2e_status() == E2eStatus::ResourceBlocked
6478                || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6479        }) {
6480            let shell = self.state.config.shell();
6481            return self
6482                .stop_unresolved_review(
6483                    "the last round's own verification never resolved",
6484                    &shell,
6485                    &winner.worktree,
6486                )
6487                .await;
6488        }
6489
6490        self.recovery_repair(&winner).await?;
6491        if self.resource_halt()? {
6492            return Ok(());
6493        }
6494
6495        let repo = self.state.repo.clone();
6496        let root = self.state.worktree_root();
6497        let language = self.state.config.graph.language.clone();
6498        let sessions = self.state.config.graph.sessions;
6499        let artifacts = self.merge_repair_artifacts();
6500        let base = self.landing_base();
6501        let base_short = short(&base);
6502        let reviewers = self.roles.reviewers.clone();
6503        let shell = self.state.config.shell();
6504
6505        for round in (self.state.reviews.len() + 1)..=max_rounds {
6506            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6507            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6508            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6509            // The prior round's own record, already persisted — never a
6510            // hand-carried variable of just its failing output: that is
6511            // exactly what let a round's e2e result drift out of sync with
6512            // which commit it was actually about (see `SCHEMA`'s doc for
6513            // schema 8). Judged against `head`, the commit reviewers are
6514            // about to look at now, so the summary always reads as "an
6515            // earlier head" here — this round's own patch has not been
6516            // checked yet.
6517            let prev_verification = self
6518                .state
6519                .reviews
6520                .last()
6521                .and_then(|r| r.verification_summary(&head));
6522            // Only the first round of a run that took a branch over: later
6523            // rounds have this run's own history.
6524            let prior_section = match (&self.state.recovery, round) {
6525                (Some(rec), 1) if self.state.reviews.is_empty() => {
6526                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6527                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6528                            Some(crate::recovery::head_of(
6529                                &git::diff(&winner.worktree, reviewed, "HEAD")
6530                                    .await
6531                                    .unwrap_or_default(),
6532                                crate::recovery::SINCE_DIFF_MAX,
6533                            ))
6534                        }
6535                        _ => None,
6536                    };
6537                    Some(prompt::recovery_review(rec, since.as_deref()))
6538                }
6539                _ => None,
6540            };
6541
6542            // Each reviewer gets its own detached checkout of exactly this
6543            // commit: nobody can perturb the winner's tree, and the fixer can
6544            // keep working without racing a reviewer.
6545            let mut jobs = Vec::new();
6546            let coordination = self.coordinate(&repo).await?;
6547            for (r, spec) in reviewers.iter().cloned().enumerate() {
6548                let wt = root.join(format!("review-{}", r + 1));
6549                // `clean -fdx` in a reset wipes setup's products, so every
6550                // round sets the seat up again.
6551                if wt.exists() {
6552                    git::reset_detached(&wt, &head).await?;
6553                } else {
6554                    git::worktree_add_detached(&repo, &wt, &head).await?;
6555                }
6556                self.setup_seat_worktree(&repo, &wt, false).await?;
6557                let seat_key = format!("review-{}", r + 1);
6558                // The seat starts the round on whoever answered it last, not
6559                // on the agent the spec names, so a failure is not re-paid.
6560                let spec = pick_start_spec(
6561                    &self.roles.reviewer_roster,
6562                    spec,
6563                    self.state.seat_history.get(&seat_key),
6564                );
6565                let seat = self.seat(&seat_key, &spec.id);
6566                jobs.push(SeatJob {
6567                    prompt: prompt::review(&prompt::ReviewCtx {
6568                        instruction: &self.state.instruction,
6569                        branch: &winner.branch,
6570                        base_short: &base_short,
6571                        stat: &stat,
6572                        patch: &patch,
6573                        verification: prev_verification.as_ref(),
6574                        prior: prior_section.as_deref(),
6575                        reviewers: reviewers.len(),
6576                        round,
6577                        rounds: max_rounds,
6578                        // A review-only run has no rankings, so nothing
6579                        // competed for this patch and the reviewer is told so.
6580                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6581                        lens: Lens::for_seat(r),
6582                        language: &language,
6583                        github_language: &self.state.config.graph.github_language,
6584                    }),
6585                    spec,
6586                    seat,
6587                    cwd: wt,
6588                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6589                    allow_write: false,
6590                    sessions,
6591                    artifacts: artifacts.clone(),
6592                    stem: format!("review-{round}-{}", r + 1),
6593                    handover: None,
6594                });
6595            }
6596
6597            drop(coordination);
6598            self.state.event(
6599                "review",
6600                format!(
6601                    "round {round}: {} reviewers on {}",
6602                    jobs.len(),
6603                    short(&head)
6604                ),
6605            );
6606            let mut quota_losses = Vec::new();
6607            let review_retries = self.state.config.graph.retries;
6608            let review_cache = self.state.config.cache_dir();
6609            let ctx = WaveCtx {
6610                carry_seats: true,
6611                pauses: Some((&self.pause, &self.interrupt)),
6612                run: &run_id,
6613                node: "review",
6614                prompts: &prompts,
6615                cache: review_cache.as_deref(),
6616                round: Some(round),
6617            };
6618            let results = ask_json_wave::<Review>(
6619                jobs,
6620                Arc::clone(&self.sem),
6621                review_retries,
6622                &self.roles.reviewer_roster,
6623                &ctx,
6624                &mut quota_losses,
6625                &mut self.state,
6626                &|_: &Review| Ok(()),
6627            )
6628            .await;
6629            if self.state.resource_block.is_some() {
6630                return self.stop_for_resource();
6631            }
6632            // Counted before the move below: how many of *this* round's
6633            // reviewer seats were lost to their own rate limit, as opposed to
6634            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6635            let round_quota_missing = quota_losses.len();
6636            self.state.quota.extend(quota_losses);
6637
6638            let mut records = Vec::new();
6639            let mut all_findings = Vec::new();
6640            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6641                let agent_id = seat.agent.clone();
6642                self.state.seats.insert(seat.key.clone(), seat);
6643                let mut record = ReviewRecord {
6644                    reviewer: r + 1,
6645                    agent: agent_id,
6646                    summary: String::new(),
6647                    findings: Vec::new(),
6648                    vote: None,
6649                    failed: None,
6650                    duration_ms: 0,
6651                    // Set for both outcomes: `failed: Some(_)` with
6652                    // `attempts > 0` is a seat every retry still lost, not a
6653                    // recovered one — only `failed: None` with `attempts > 0`
6654                    // reads as "answered after a nudge" (see this field's own
6655                    // doc).
6656                    attempts,
6657                };
6658                match res {
6659                    Ok((review, out)) => {
6660                        // Sanitized here, at the point every other piece of
6661                        // agent prose in this file is (candidate summaries,
6662                        // deliberation turns, vote reasons): a reviewer's own
6663                        // words are the one thing about it that could name
6664                        // it, and reconsideration below broadcasts this same
6665                        // summary and these same findings to every other
6666                        // seat on the panel.
6667                        record.summary =
6668                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6669                        record.vote = Some(review.vote);
6670                        record.duration_ms = out.duration_ms;
6671                        for (n, mut f) in review.findings.into_iter().enumerate() {
6672                            // ids are magi's, never the agent's: the fixer's
6673                            // adoption report is keyed by them.
6674                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6675                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6676                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6677                            // `file` is agent-supplied prose too, never
6678                            // checked against the real tree — the same
6679                            // exposure `title`/`detail` above have, just in
6680                            // a field easy to forget because it looks like a
6681                            // path rather than free text.
6682                            f.file = f
6683                                .file
6684                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6685                            all_findings.push(f.clone());
6686                            record.findings.push(f);
6687                        }
6688                        self.state.event(
6689                            "review",
6690                            format!(
6691                                "round {round}: reviewer {} voted {} with {} finding(s)",
6692                                r + 1,
6693                                review.vote.label(),
6694                                record.findings.len()
6695                            ),
6696                        );
6697                    }
6698                    Err(e) => {
6699                        record.failed = Some(e.to_string());
6700                        self.state.event(
6701                            "review",
6702                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6703                        );
6704                    }
6705                }
6706                records.push(record);
6707            }
6708
6709            // Tally the round's votes and, if they split, spend the one
6710            // round of reconsideration the split -> deliberate -> revote
6711            // shape `judge`/`vote` use for the panel, sized down to what a
6712            // read-only review round can afford: one round, and a revote
6713            // rather than an argument, because the panel already wrote its
6714            // reasoning down as findings the first time around.
6715            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6716            let vote_split =
6717                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6718            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6719            if vote_split {
6720                self.state.event(
6721                    "review",
6722                    format!(
6723                        "round {round}: votes split ({}) — one round of reconsideration",
6724                        initial_votes
6725                            .iter()
6726                            .map(|v| v.label())
6727                            .collect::<Vec<_>>()
6728                            .join(", ")
6729                    ),
6730                );
6731                // Seats read every seat's findings and votes, still numbered
6732                // and never named — the same anonymity `review` itself keeps.
6733                let panel: Vec<ReviewSeatReport<'_>> = records
6734                    .iter()
6735                    .filter_map(|r| {
6736                        r.vote.map(|vote| ReviewSeatReport {
6737                            reviewer: r.reviewer,
6738                            vote,
6739                            summary: &r.summary,
6740                            findings: &r.findings,
6741                        })
6742                    })
6743                    .collect();
6744
6745                let mut jobs = Vec::new();
6746                let mut seats_at = Vec::new();
6747                for (r, spec) in reviewers.iter().cloned().enumerate() {
6748                    // A seat with no initial vote has nothing to reconsider
6749                    // from and stays absent, the same as it stayed absent
6750                    // from `panel` above.
6751                    if records[r].vote.is_none() {
6752                        continue;
6753                    }
6754                    let wt = root.join(format!("review-{}", r + 1));
6755                    let seat_key = format!("review-{}", r + 1);
6756                    let spec = self.occupant(&seat_key, spec);
6757                    let seat = self.seat(&seat_key, &spec.id);
6758                    // A seat with no live session has already forgotten the
6759                    // initial review's prompt — restate the patch it is
6760                    // voting on, the same as `deliberate`/`vote` do for a
6761                    // judge in the same position.
6762                    // The panel already carries this seat's own review and
6763                    // vote, so restating the patch makes the prompt whole for
6764                    // a seat handed to another agent.
6765                    let build = |with_patch: bool| {
6766                        prompt::review_reconsider(&ReviewReconsiderCtx {
6767                            instruction: &self.state.instruction,
6768                            reviewer: r + 1,
6769                            lens: Lens::for_seat(r),
6770                            panel: &panel,
6771                            patch: with_patch.then_some(ReviewPatch {
6772                                branch: &winner.branch,
6773                                base_short: &base_short,
6774                                stat: &stat,
6775                                patch: &patch,
6776                            }),
6777                            round,
6778                            rounds: max_rounds,
6779                            language: &language,
6780                        })
6781                    };
6782                    let full = build(true);
6783                    let prompt = if has_context(&spec, &seat, sessions) {
6784                        build(false)
6785                    } else {
6786                        full.clone()
6787                    };
6788                    jobs.push(SeatJob {
6789                        prompt,
6790                        spec,
6791                        seat,
6792                        cwd: wt,
6793                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6794                        allow_write: false,
6795                        sessions,
6796                        artifacts: artifacts.clone(),
6797                        stem: format!("review-{round}-reconsider-{}", r + 1),
6798                        handover: Some(full),
6799                    });
6800                    seats_at.push(r);
6801                }
6802
6803                let mut recon_quota_losses = Vec::new();
6804                let recon_cache = self.state.config.cache_dir();
6805                let recon_ctx = WaveCtx {
6806                    carry_seats: true,
6807                    pauses: Some((&self.pause, &self.interrupt)),
6808                    run: &run_id,
6809                    node: "review",
6810                    prompts: &prompts,
6811                    cache: recon_cache.as_deref(),
6812                    round: Some(round),
6813                };
6814                let recon_results = ask_json_wave::<ReviewRevote>(
6815                    jobs,
6816                    Arc::clone(&self.sem),
6817                    review_retries,
6818                    &self.roles.reviewer_roster,
6819                    &recon_ctx,
6820                    &mut recon_quota_losses,
6821                    &mut self.state,
6822                    &|_: &ReviewRevote| Ok(()),
6823                )
6824                .await;
6825                if self.state.resource_block.is_some() {
6826                    return self.stop_for_resource();
6827                }
6828                self.state.quota.extend(recon_quota_losses);
6829
6830                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6831                    let agent_id = seat.agent.clone();
6832                    self.state.seats.insert(seat.key.clone(), seat);
6833                    let mut rec = ReviewRevoteRecord {
6834                        reviewer: r + 1,
6835                        agent: agent_id,
6836                        vote: None,
6837                        reason: String::new(),
6838                        failed: None,
6839                    };
6840                    match res {
6841                        Ok((rv, _)) => {
6842                            rec.vote = Some(rv.vote);
6843                            rec.reason =
6844                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6845                            self.state.event(
6846                                "review",
6847                                format!(
6848                                    "round {round}: reviewer {} revoted {}",
6849                                    r + 1,
6850                                    rv.vote.label()
6851                                ),
6852                            );
6853                        }
6854                        Err(e) => {
6855                            rec.failed = Some(e.to_string());
6856                            self.state.event(
6857                                "review",
6858                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6859                            );
6860                        }
6861                    }
6862                    reconsideration.push(rec);
6863                }
6864            } else if initial_votes.len() > 1 {
6865                self.state.event(
6866                    "review",
6867                    format!(
6868                        "round {round}: votes agreed ({}) — no reconsideration",
6869                        initial_votes[0].label()
6870                    ),
6871                );
6872            }
6873
6874            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6875            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6876            // A round that already has a blocking finding and a round left to
6877            // try is going back to the fixer no matter what `verify.e2e`
6878            // says, so running it first only spends the loop's slowest step
6879            // (minutes, for a Rust repo's full test suite) on a head about
6880            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6881            // runs once a round has no blocking findings left (see
6882            // `round_is_clean`, which a deferred — empty — `e2e` can never
6883            // satisfy since `blocking` is nonzero whenever this branch is
6884            // taken), and `stop_reviewing` forces a real run before it will
6885            // ever read a deferred round as green.
6886            let defer_e2e =
6887                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6888            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6889                let reason =
6890                    format!("{blocking} blocking finding(s) already required a fix this round");
6891                self.state.event(
6892                    "verify",
6893                    format!(
6894                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6895                         {}); it will run once a round has none left",
6896                        short(&head)
6897                    ),
6898                );
6899                (Vec::new(), false, true, Some(reason))
6900            } else {
6901                let e2e_commands = self.state.config.verify.e2e.clone();
6902                let cache_dir = self.state.config.cache_dir();
6903                let context = format!("round {round}");
6904                let wait = self.repository_wait();
6905                let (e2e, verify_retried) = with_cache_lease(
6906                    &mut self.state,
6907                    cache_dir.as_deref(),
6908                    "e2e",
6909                    "e2e",
6910                    &winner.worktree,
6911                    &head,
6912                    verify_timeout,
6913                    &context,
6914                    &wait,
6915                    |state, budget| {
6916                        let shell = shell.clone();
6917                        let e2e_commands = e2e_commands.clone();
6918                        let worktree = winner.worktree.clone();
6919                        let context = context.clone();
6920                        async move {
6921                            run_e2e_with_retry(
6922                                state,
6923                                &shell,
6924                                &e2e_commands,
6925                                &worktree,
6926                                budget,
6927                                &context,
6928                            )
6929                            .await
6930                        }
6931                    },
6932                )
6933                .await?;
6934                (e2e, verify_retried, false, None)
6935            };
6936
6937            let expected = records.len();
6938            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6939            let incomplete = answered < expected;
6940            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6941            let policy = self.state.config.graph.incomplete_review;
6942            let clean = round_is_clean(
6943                blocking,
6944                e2e_ok,
6945                answered,
6946                expected,
6947                round_quota_missing,
6948                policy,
6949            );
6950
6951            let mut round_record = ReviewRound {
6952                round,
6953                head: head.clone(),
6954                verified_head: None,
6955                verified_at: None,
6956                reviews: records,
6957                e2e,
6958                verify_retried,
6959                e2e_deferred,
6960                e2e_defer_reason,
6961                fix: None,
6962                blocking,
6963                answered,
6964                expected,
6965                clean,
6966                progressed: false,
6967                vote_split,
6968                reconsideration,
6969                verdict: None,
6970            };
6971            // The final vote per seat is its revote where reconsideration
6972            // ran and answered, its initial vote otherwise — the same
6973            // fallback `tally` uses for a judge whose private vote failed.
6974            round_record.verdict = ReviewVote::worst(
6975                round_record
6976                    .final_votes()
6977                    .into_iter()
6978                    .map(|(_, _, vote)| vote),
6979            );
6980            // Which commit and when magi actually attempted to check —
6981            // known the moment a command was dispatched against `head`,
6982            // whether or not it finished: a resource-blocked attempt still
6983            // targeted a specific commit at a specific time, and leaving
6984            // that unrecorded is exactly what made `verification_summary`
6985            // report a fresh attempt as "commit unknown ... recorded before
6986            // this was tracked", indistinguishable from a genuinely old,
6987            // untracked record. Only a deferred or unconfigured round never
6988            // ran at all and has nothing to record — see
6989            // `ReviewRound::verified_head`'s own doc.
6990            if !matches!(
6991                round_record.e2e_status(),
6992                E2eStatus::Deferred | E2eStatus::NotConfigured
6993            ) {
6994                round_record.verified_head = Some(head.clone());
6995                round_record.verified_at = Some(Timestamp::now());
6996            }
6997
6998            if incomplete {
6999                let missing: Vec<String> = round_record
7000                    .reviews
7001                    .iter()
7002                    .filter(|r| r.failed.is_some())
7003                    .map(|r| format!("review-{}", r.reviewer))
7004                    .collect();
7005                self.state.event(
7006                    "review",
7007                    format!(
7008                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7009                        missing.join(", ")
7010                    ),
7011                );
7012            }
7013
7014            if clean {
7015                self.state.event(
7016                    "review",
7017                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7018                        format!(
7019                            "round {round}: clean (warn policy, incomplete panel) — no \
7020                             blocking findings from the seats that answered, verification green"
7021                        )
7022                    } else if incomplete {
7023                        format!(
7024                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7025                             quorum) — no blocking findings from the seats that answered, \
7026                             verification green",
7027                            expected - answered
7028                        )
7029                    } else {
7030                        format!("round {round}: clean — no blocking findings, verification green")
7031                    },
7032                );
7033                self.state.reviews.push(round_record);
7034                self.state.status = RunStatus::Gating;
7035                self.state.save()?;
7036                return Ok(());
7037            }
7038
7039            // Nothing was raised and verification passed, but not every seat
7040            // answered and `round_is_clean` still refused to call it clean —
7041            // either a seat is missing for a reason other than its own quota
7042            // (a crash, a timeout, unparsable output — worth another try), or
7043            // every seat that could have answered lost its quota and nobody
7044            // is left to decide on: re-review rather than send the fixer
7045            // after a round with nothing to fix.
7046            if incomplete && blocking == 0 && e2e_ok {
7047                self.state.reviews.push(round_record);
7048                self.state.save()?;
7049                if round == max_rounds {
7050                    self.state.status = RunStatus::Blocked;
7051                    self.state.event(
7052                        "review",
7053                        format!(
7054                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7055                             refusing to call it clean",
7056                            expected - answered
7057                        ),
7058                    );
7059                    return Ok(());
7060                }
7061                continue;
7062            }
7063
7064            // Nothing for the fixer to act on (`blocking == 0`) and the only
7065            // reason this round is not clean is that magi itself never got
7066            // a command to run — the shared build cache, not the patch (see
7067            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7068            // the fixer would invite a change to appease contention that has
7069            // nothing to do with the diff, and would leave this attempt
7070            // sitting in the next round's prompt as if it were about an
7071            // earlier, superseded commit rather than what it actually is:
7072            // the same head, still waiting to be checked. Wait for it the
7073            // same way the final round's own contention is already handled,
7074            // whatever round this happens to be.
7075            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7076                self.state.reviews.push(round_record);
7077                return self
7078                    .stop_unresolved_review(
7079                        "the round's own verification could not run",
7080                        &shell,
7081                        &winner.worktree,
7082                    )
7083                    .await;
7084            }
7085
7086            if round == max_rounds {
7087                self.state.reviews.push(round_record);
7088                return self
7089                    .stop_reviewing(
7090                        &format!(
7091                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7092                        ),
7093                        &shell,
7094                        &winner.worktree,
7095                    )
7096                    .await;
7097            }
7098
7099            if !self.fix_review_round(&winner, round_record).await? {
7100                return Ok(());
7101            }
7102        }
7103        Ok(())
7104    }
7105
7106    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7107    /// False means admission or continuation policy stopped the review loop.
7108    async fn fix_review_round(
7109        &mut self,
7110        winner: &Candidate,
7111        mut round_record: ReviewRound,
7112    ) -> Result<bool> {
7113        let round = round_record.round;
7114        let shell = self.state.config.shell();
7115        let step = self.run_fix_round(winner, &mut round_record).await?;
7116        if matches!(step, FixStep::Resource) {
7117            self.stop_for_resource()?;
7118            return Ok(false);
7119        }
7120        if self.state.reviews.len() < round {
7121            self.state.reviews.push(round_record);
7122        } else {
7123            self.state.reviews[round - 1] = round_record;
7124        }
7125        self.state.save()?;
7126
7127        // The fixer's own report never came back this round, even after
7128        // `continue_fix_report`'s own budget was spent on it — not an
7129        // ordinary "no report" (dropped stream, quota, plain failure),
7130        // which already reads that way and is left to the existing round
7131        // budget. Stopping here, rather than opening another round, is
7132        // what keeps a next reviewer/fixer wave from ever being
7133        // dispatched onto `winner.worktree` while whatever the seat's
7134        // last call may still have running there is unaccounted for: no
7135        // process liveness check exists (and none is being added — see
7136        // AGENTS.md/this task's own scope), so the only way to honour
7137        // "nothing starts before a valid report returns" is to not start
7138        // anything further on this worktree from this run at all.
7139        if matches!(step, FixStep::Recorded { report_lost: true }) {
7140            self.stop_reviewing(
7141                "the fixer's adoption report never came back, even after resuming its \
7142                         own seat; refusing to start another round against the same worktree \
7143                         while that is unresolved",
7144                &shell,
7145                &winner.worktree,
7146            )
7147            .await?;
7148            return Ok(false);
7149        }
7150
7151        let streak = self
7152            .state
7153            .reviews
7154            .iter()
7155            .rev()
7156            .take_while(|r| !r.progressed)
7157            .count();
7158        if streak >= STAGNANT_LIMIT {
7159            self.stop_reviewing(
7160                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7161                &shell,
7162                &winner.worktree,
7163            )
7164            .await?;
7165            return Ok(false);
7166        }
7167        Ok(true)
7168    }
7169
7170    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7171    async fn run_fix_round(
7172        &mut self,
7173        winner: &Candidate,
7174        round_record: &mut ReviewRound,
7175    ) -> Result<FixStep> {
7176        let round = round_record.round;
7177        let max_rounds = self.state.config.graph.review_rounds;
7178        let run_id = self.state.id.clone();
7179        let prompts = self.state.config.prompts.clone();
7180        let language = self.state.config.graph.language.clone();
7181        let sessions = self.state.config.graph.sessions;
7182        let artifacts = self.merge_repair_artifacts();
7183        let base = self.landing_base();
7184        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7185        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7186        let this_round_verification = round_record.verification_summary(&head);
7187        let all_findings: Vec<_> = round_record
7188            .reviews
7189            .iter()
7190            .flat_map(|r| r.findings.iter().cloned())
7191            .collect();
7192        // Fix. The winner's own implementer seat continues its conversation:
7193        // the competition is over, so context is pure benefit now.
7194        let blocking_findings: Vec<_> = all_findings
7195            .iter()
7196            .filter(|f| f.severity.blocks())
7197            .cloned()
7198            .collect();
7199        let fix_prompt = prompt::fix_in(
7200            &self.state.instruction,
7201            &blocking_findings,
7202            this_round_verification.as_ref(),
7203            round,
7204            max_rounds,
7205            &language,
7206            &self.state.config.graph.github_language,
7207        );
7208        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7209        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7210        let (job, seat, out) = self
7211            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7212                prompt: fix_prompt.clone(),
7213                spec,
7214                seat,
7215                cwd: winner.worktree.clone(),
7216                timeout,
7217                allow_write: true,
7218                sessions,
7219                artifacts: artifacts.clone(),
7220                stem: format!("fix-{round}"),
7221                handover: None,
7222            })
7223            .await;
7224        // Not asked: the round is not recorded (so a resume repeats it),
7225        // no fixer was handed over to, and the seat is left as it was.
7226        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7227            return Ok(FixStep::Resource);
7228        }
7229        let agent_id = seat.agent.clone();
7230
7231        let mut fix = FixRecord {
7232            agent: agent_id,
7233            addressed: Vec::new(),
7234            rejected: Vec::new(),
7235            notes: String::new(),
7236            committed: false,
7237            failed: None,
7238            duration_ms: 0,
7239            continuation: None,
7240        };
7241        let mut continuation = ContinuationRecord::not_needed();
7242        let mut final_seat = seat.clone();
7243        match out {
7244            AgentOutcome::Ok(o) => {
7245                fix.duration_ms = o.duration_ms;
7246                let parsed = verdict::extract_json::<FixReport>(&o.text);
7247                // A parsed report standing next to a command this same
7248                // reply's own CLI never confirmed the exit status of is
7249                // not a resolved answer — the identical `CommandEvidence`
7250                // `state.jobs` renders, read here instead of only on
7251                // display, per the completion judgment and the shown
7252                // record needing to agree.
7253                let incomplete_reason = match &parsed {
7254                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7255                        "the reply parsed, but it reported a command whose own CLI never \
7256                             confirmed an exit status"
7257                            .to_owned(),
7258                    ),
7259                    Ok(_) => None,
7260                    Err(e) => Some(e.to_string()),
7261                };
7262                match incomplete_reason {
7263                    None => {
7264                        let report = parsed.expect("checked Ok above");
7265                        fix.addressed = report.addressed;
7266                        fix.rejected = report.rejected;
7267                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7268                    }
7269                    Some(reason) => {
7270                        let (resumed_seat, resolved, failure, cont) = self
7271                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7272                            .await;
7273                        fix.duration_ms += cont.cumulative_wait_ms;
7274                        continuation = cont;
7275                        final_seat = resumed_seat;
7276                        match resolved {
7277                            Some(report) => {
7278                                fix.addressed = report.addressed;
7279                                fix.rejected = report.rejected;
7280                                fix.notes =
7281                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
7282                            }
7283                            None => fix.failed = failure,
7284                        }
7285                    }
7286                }
7287            }
7288            // The CLI's raw error JSON is not a fix report to parse.
7289            AgentOutcome::Dropped(o) => {
7290                fix.duration_ms = o.duration_ms;
7291                let why = o
7292                    .dropped
7293                    .as_ref()
7294                    .map(|d| d.why.as_str())
7295                    .unwrap_or("the CLI ended the stream without delivering its answer");
7296                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7297            }
7298            AgentOutcome::Quota(o) => {
7299                self.state.quota.push(QuotaLoss::from_report(
7300                    final_seat.key.clone(),
7301                    "fix",
7302                    o.quota.as_ref(),
7303                ));
7304                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7305            }
7306            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7307        }
7308        if self.state.resource_block.is_some() {
7309            self.state.seats.insert(final_seat.key.clone(), final_seat);
7310            return Ok(FixStep::Resource);
7311        }
7312        fix.continuation = Some(continuation);
7313        self.state.seats.insert(final_seat.key.clone(), final_seat);
7314        let coordination = self.coordinate(&winner.worktree).await?;
7315        if let Ok(r) = git::rescue_commit(
7316            &winner.worktree,
7317            &format!("magi: review round {round} fixes (uncommitted work)"),
7318        )
7319        .await
7320        {
7321            self.state.note_withheld("fix", &r.withheld);
7322        }
7323        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7324        fix.committed = after != before;
7325        // Judged by what `git` says moved against base, never by the
7326        // fixer's own `addressed`/`rejected` count — see
7327        // `ReviewRound::progressed`. Propagated with `?`, the same as the
7328        // `patch` snapshot above: swallowing this error would default
7329        // `diff_after` to empty, which almost always differs from a
7330        // non-empty `patch` and reads as "progressed" — exactly backwards
7331        // for a `git` failure the stagnation check cannot see through.
7332        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7333        drop(coordination);
7334        let progressed = diff_after != patch;
7335        let commit_note = if fix.committed {
7336            "committed"
7337        } else {
7338            "NO new commit"
7339        };
7340        let tree_note = if progressed {
7341            "changed vs base"
7342        } else {
7343            "unchanged vs base"
7344        };
7345        self.state.event(
7346            "fix",
7347            match &fix.failed {
7348                // Distinct on purpose from "0 addressed, 0 rejected": the
7349                // fixer's own diff still landed (blocking counts do keep
7350                // falling round over round), only its adoption report did
7351                // not come back, so this must never read like every
7352                // finding was reviewed and declined.
7353                Some(reason) => {
7354                    format!(
7355                        "round {round}: fixer's adoption report was lost ({reason}); \
7356                             {commit_note}, tree {tree_note}"
7357                    )
7358                }
7359                None => format!(
7360                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7361                         {tree_note}{}",
7362                    fix.addressed.len(),
7363                    fix.rejected.len(),
7364                    if continuation.outcome == ContinuationOutcome::Resumed {
7365                        format!(
7366                            " (adoption report recovered after {} continuation(s))",
7367                            continuation.attempts
7368                        )
7369                    } else {
7370                        String::new()
7371                    },
7372                ),
7373            },
7374        );
7375        round_record.fix = Some(fix);
7376        round_record.progressed = progressed;
7377        Ok(FixStep::Recorded {
7378            report_lost: matches!(
7379                continuation.outcome,
7380                ContinuationOutcome::Exhausted
7381                    | ContinuationOutcome::QuotaLost
7382                    | ContinuationOutcome::NoSession
7383            ),
7384        })
7385    }
7386
7387    /// Decide, from the last recorded round's own verification, whether
7388    /// stopping the review loop is a hand-off or a genuine block.
7389    ///
7390    /// Called once the loop has given up trying — the round budget is spent,
7391    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
7392    /// findings still open, never while a round is still clean or the
7393    /// incomplete-panel case handled inline above. Gate and e2e are facts
7394    /// about the tree; a lingering review finding is an opinion, and this
7395    /// workload's own `magi stats` puts reviewer precision low enough
7396    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
7397    /// must not by itself stand between a green, verified change and the
7398    /// human who decides what to do with it. A red e2e is not an opinion, so
7399    /// that case still blocks, with the failing command and a tail of its
7400    /// output recorded here rather than left in `run.json` for someone to go
7401    /// find.
7402    ///
7403    /// A round that deferred its own e2e (see [`Config::graph`]'s
7404    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
7405    /// only because nothing ran, and treating an empty list as a passing one
7406    /// here is exactly the "deferred painted green" bug this function exists
7407    /// to not have. When the last round's own verification never resolved —
7408    /// deferred on purpose, or a real attempt the shared build cache blocked
7409    /// — this makes (or retries) the real run, on the actual worktree this
7410    /// loop is about to stop touching, before deciding anything. A
7411    /// resource-blocked attempt is likewise never read as either green or
7412    /// red: it is evidence about the machine, not the patch (see
7413    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
7414    /// blocked cache leaves this call without deciding rather than guessing
7415    /// — the caller retries on a later reentry.
7416    /// Record, once, that the review loop handed off over a blocking finding
7417    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
7418    /// `land` asks the owner even with `land_approval` off. Called from every
7419    /// path that concludes `Gating`; a reentry keeps the first record.
7420    fn record_contested_handoff(&mut self) {
7421        if self.state.contested_handoff.is_some() {
7422            return;
7423        }
7424        let Some(contested) = self
7425            .state
7426            .reviews
7427            .last()
7428            .and_then(ReviewRound::contested_handoff)
7429        else {
7430            return;
7431        };
7432        self.state.event(
7433            "review",
7434            format!(
7435                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7436                 wait for the owner's approval",
7437                contested.findings.len(),
7438                contested.rejecters.len()
7439            ),
7440        );
7441        self.state.contested_handoff = Some(contested);
7442    }
7443
7444    async fn stop_unresolved_review(
7445        &mut self,
7446        why: &str,
7447        shell: &[String],
7448        worktree: &Path,
7449    ) -> Result<()> {
7450        self.stop_reviewing_inner(why, shell, worktree, true).await
7451    }
7452
7453    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7454        self.stop_reviewing_inner(why, shell, worktree, false).await
7455    }
7456
7457    async fn stop_reviewing_inner(
7458        &mut self,
7459        why: &str,
7460        shell: &[String],
7461        worktree: &Path,
7462        repairable: bool,
7463    ) -> Result<()> {
7464        let round_idx = self.state.reviews.len() - 1;
7465        // A deferred round and a resource-blocked one are the same shape
7466        // here: neither has a real result yet, and both get one more
7467        // attempt. Read off `e2e_status` — the single source for this —
7468        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
7469        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
7470        // still retries instead of being read as a settled result the
7471        // instant it stops being empty.
7472        let needs_catchup_run = matches!(
7473            self.state.reviews[round_idx].e2e_status(),
7474            E2eStatus::Deferred | E2eStatus::ResourceBlocked
7475        );
7476        if needs_catchup_run {
7477            let round = self.state.reviews[round_idx].round;
7478            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7479            let commands = self.state.config.verify.e2e.clone();
7480            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7481            let cache_dir = self.state.config.cache_dir();
7482            let context = format!(
7483                "round {round}: verification unresolved, catching up before the final decision"
7484            );
7485            let wait = self.repository_wait();
7486            let (outcomes, verify_retried) = with_cache_lease(
7487                &mut self.state,
7488                cache_dir.as_deref(),
7489                "e2e",
7490                "e2e",
7491                worktree,
7492                &attempted_head,
7493                timeout,
7494                &context,
7495                &wait,
7496                |state, budget| {
7497                    let shell = shell.to_vec();
7498                    let commands = commands.clone();
7499                    let context = context.clone();
7500                    async move {
7501                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7502                            .await
7503                    }
7504                },
7505            )
7506            .await?;
7507            let last = &mut self.state.reviews[round_idx];
7508            last.e2e = outcomes;
7509            last.verify_retried = verify_retried;
7510            // Always the commit and time this attempt actually targeted,
7511            // whether or not it happens to equal the reviewed `head` and
7512            // whether or not a command finished — see
7513            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
7514            // attempt is recorded too, so a later reader sees "attempted
7515            // again at T2" rather than silence.
7516            last.verified_head = Some(attempted_head);
7517            last.verified_at = Some(Timestamp::now());
7518            if verify_inconclusive(&last.e2e) {
7519                // Still not a real result: `e2e_deferred` is left exactly
7520                // as it was, so `needs_catchup_run` above reads
7521                // `ResourceBlocked` (via `e2e_status`, which checks
7522                // `resource_blocked` before `e2e_deferred`) and retries
7523                // again on the next reentry, rather than recording
7524                // contention as a red e2e and blocking the run on it.
7525                self.state.save()?;
7526                return Ok(());
7527            }
7528            last.e2e_deferred = false;
7529        }
7530        let last = &self.state.reviews[round_idx];
7531        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7532
7533        match last.e2e_status() {
7534            E2eStatus::Failed => {
7535                let red: Vec<String> = last
7536                    .e2e
7537                    .iter()
7538                    .filter(|o| !o.ok())
7539                    .map(|o| {
7540                        format!(
7541                            "`{}` -> {:?}\n{}",
7542                            o.command,
7543                            o.code,
7544                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7545                        )
7546                    })
7547                    .collect();
7548                let can_repair = last.blocking == 0 && last.fix.is_none();
7549                self.state.reviews[round_idx].clean = false;
7550                self.state
7551                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7552                self.state.status = RunStatus::Blocked;
7553                if repairable
7554                    && can_repair
7555                    && self.state.reviews.len() < self.state.config.graph.review_rounds
7556                {
7557                    self.state.status = RunStatus::Reviewing;
7558                }
7559            }
7560            // `needs_catchup_run` above already retried once this call; if
7561            // it is still blocked, this is magi's own admission it could
7562            // not get a command to run, never a verdict on the patch — the
7563            // run is left exactly where a later reentry can retry again.
7564            E2eStatus::ResourceBlocked => {
7565                self.state.event(
7566                    "review",
7567                    format!(
7568                        "{why}; e2e could not run (resource unavailable); not \
7569                         deciding yet"
7570                    ),
7571                );
7572            }
7573            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7574                self.state.event(
7575                    "review",
7576                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7577                );
7578                self.record_contested_handoff();
7579                self.state.status = RunStatus::Gating;
7580            }
7581        }
7582        self.state.save()?;
7583        Ok(())
7584    }
7585
7586    // ----------------------------------------------------------------- gate
7587
7588    async fn gate(&mut self) -> Result<()> {
7589        if self.hold_base_sync().await? {
7590            return Ok(());
7591        }
7592        // Judged by the review record itself, not by `status`: a solo
7593        // candidate's `judge`/`deliberate` skip rewrites `status` on every
7594        // reentry (see `judge`), and trusting it here is exactly how a run
7595        // that exhausted its review budget got gated and merged a second
7596        // time around. `review_conclusion` recomputes the review loop's own
7597        // verdict from the round records themselves — `Gating` for a clean
7598        // round or a hand-off (see `stop_reviewing`), anything else means the
7599        // loop is still going or genuinely blocked.
7600        // A base the winner could not be replayed onto is a decision, not a
7601        // round: there is no landing tree to gate. Read as its own record for
7602        // the same reason the review verdict is.
7603        if self.state.status == RunStatus::Failed
7604            || self
7605                .state
7606                .base_sync
7607                .as_ref()
7608                .is_some_and(|s| s.conflict.is_some())
7609            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7610                != Some(RunStatus::Gating)
7611        {
7612            return Ok(());
7613        }
7614        let Some(winner) = self.state.winner().cloned() else {
7615            return Ok(());
7616        };
7617        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7618        self.state.reconcile_verification_head(&head);
7619        self.state.save()?;
7620        let wait = self.repository_wait();
7621        if !verify_current_head(&mut self.state, &winner, &wait).await? {
7622            return Ok(());
7623        }
7624        if self.state.gate_ran {
7625            // `review_loop` derives its conclusion from the clean review
7626            // record on every reentry and therefore puts a completed run back
7627            // in `Gating`. A recorded gate is a stronger, terminal fact:
7628            // retain its original command output (or lack of any, for a repo
7629            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
7630            // doc) and restore `Blocked` on a real failure rather than
7631            // pretending the command is still running or running it a second
7632            // time. `gate_ran == false` remains the only shape — unattempted,
7633            // or a resource-blocked retry — that may still need to execute a
7634            // command.
7635            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7636                self.state.status = RunStatus::Blocked;
7637                self.state.save()?;
7638            }
7639            return Ok(());
7640        }
7641        self.state.status = RunStatus::Gating;
7642        let mut outcomes = self.run_gate(&winner).await?;
7643        let mut attempted_head = self.state.gate_head.clone();
7644        loop {
7645            // A resource-blocked outcome means the gate command never actually
7646            // ran - the shared build cache could not be acquired or confirmed
7647            // fresh in time - which is evidence about the machine, not about
7648            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
7649            // Recording it as a red gate would mark a run `Blocked` on nothing
7650            // but contention magi has already logged; leaving `self.state.gate`
7651            // empty and `self.state.gate_ran` false instead keeps the shape
7652            // this function already treats as "still needs to run" (see the
7653            // early-return above), so the next call retries the command
7654            // rather than concluding anything.
7655            if verify_inconclusive(&outcomes) {
7656                self.state.save()?;
7657                return Ok(());
7658            }
7659            if outcomes.iter().all(CommandOutcome::ok) {
7660                break;
7661            }
7662            let fixes_before = self.state.gate_fixes.len();
7663            match self.gate_fix_round(&winner, &outcomes).await? {
7664                GateFix::Retry => {
7665                    outcomes = self.run_gate(&winner).await?;
7666                    attempted_head = self.state.gate_head.clone();
7667                }
7668                GateFix::Stop => {
7669                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
7670                    if self.state.gate_fixes.len() > fixes_before
7671                        && attempted_head.as_deref() != Some(current.as_str())
7672                        && verify_current_head(&mut self.state, &winner, &wait).await?
7673                    {
7674                        // A new SHA owes a gate even when a spent fixer changed only
7675                        // commit metadata. This does not grant another repair round.
7676                        outcomes = self.run_gate(&winner).await?;
7677                        attempted_head = self.state.gate_head.clone();
7678                    } else {
7679                        break;
7680                    }
7681                }
7682                GateFix::Defer => {
7683                    self.state.save()?;
7684                    return Ok(());
7685                }
7686            }
7687        }
7688        let passed = outcomes.iter().all(CommandOutcome::ok);
7689        self.state.gate = outcomes;
7690        self.state.gate_ran = true;
7691        self.state.gate_head = attempted_head;
7692        self.state.save()?;
7693        if !verify_current_head(&mut self.state, &winner, &wait).await? {
7694            return Ok(());
7695        }
7696        if !passed {
7697            self.state.status = RunStatus::Blocked;
7698            let spent = self.state.gate_fixes.len();
7699            self.state.event(
7700                "gate",
7701                if spent == 0 {
7702                    "gate failed; not merging".to_owned()
7703                } else {
7704                    format!("gate failed after {spent} gate-fix round(s); not merging")
7705                },
7706            );
7707        }
7708        self.state.save()?;
7709        Ok(())
7710    }
7711
7712    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7713    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7714    /// after review is clean and never on a candidate awaiting judging.
7715    ///
7716    /// Never fails the run: a non-zero exit or timeout is a warning and a
7717    /// recorded outcome, and the gate remains the single arbiter. Nothing
7718    /// configured means nothing happens - no event, no commit. `commit_all`
7719    /// commits any leftover change under the neutral identity and returns
7720    /// `false` when the tree is clean, so no empty commit is ever made.
7721    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7722        let commands = state.config.verify.pre_gate.clone();
7723        if commands.is_empty() {
7724            return;
7725        }
7726        let shell = state.config.shell();
7727        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7728        let (outcomes, _) = run_commands(
7729            state,
7730            "pre_gate",
7731            "pre_gate",
7732            0,
7733            &shell,
7734            &commands,
7735            &winner.worktree,
7736            timeout,
7737        )
7738        .await;
7739        for o in &outcomes {
7740            if !o.ok() {
7741                tracing::warn!(
7742                    "pre_gate `{}` failed ({:?}); the gate decides",
7743                    o.command,
7744                    o.code
7745                );
7746            }
7747            state.event(
7748                "pre_gate",
7749                format!(
7750                    "`{}` -> {}",
7751                    o.command,
7752                    if o.ok() {
7753                        "pass".to_owned()
7754                    } else {
7755                        format!(
7756                            "FAIL ({:?})\n{}",
7757                            o.code,
7758                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7759                        )
7760                    }
7761                ),
7762            );
7763        }
7764        state.pre_gate = outcomes;
7765        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7766            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7767                Ok(head) => {
7768                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7769                    state.pre_gate_commit = Some(head);
7770                }
7771                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7772            },
7773            Ok(false) => {}
7774            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7775        }
7776        if let Err(e) = state.save() {
7777            tracing::warn!("could not persist the pre_gate record: {e:#}");
7778        }
7779    }
7780
7781    /// Run `verify.gate` once against the winner's current tree, logging one
7782    /// event per command. Empty when nothing is configured.
7783    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7784        let wait = self.repository_wait();
7785        run_landing_gate(&mut self.state, winner, &wait).await
7786    }
7787
7788    /// One bounded fix round for a failing gate.
7789    ///
7790    /// The fixer is told the failure came from the gate itself, not from a
7791    /// reviewer, and is shown the failed commands, their exit codes and a tail
7792    /// of their output - whatever `[verify].gate` holds, nothing here knows
7793    /// what those commands run. Only a normal non-zero exit that printed
7794    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7795    /// command or a full disk says nothing about the code, and a fixer sent
7796    /// after it can only appease the machine. The round is judged by what git
7797    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7798    /// again before the gate does, so a fix cannot trade a green gate for a
7799    /// red e2e unnoticed.
7800    async fn gate_fix_round(
7801        &mut self,
7802        winner: &Candidate,
7803        outcomes: &[CommandOutcome],
7804    ) -> Result<GateFix> {
7805        let cap = self.state.config.graph.gate_fix_rounds;
7806        let spent = self.state.gate_fixes.len();
7807        if spent >= cap {
7808            if cap > 0 {
7809                self.state.event(
7810                    "gate",
7811                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7812                );
7813            }
7814            return Ok(GateFix::Stop);
7815        }
7816        if !gate_fixable(outcomes) {
7817            self.state.event(
7818                "gate",
7819                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7820                 command or similar); not spending a fix round on it",
7821            );
7822            return Ok(GateFix::Stop);
7823        }
7824        let min_free = self.state.config.disk.min_free_bytes;
7825        if min_free > 0 {
7826            match crate::disk::free_bytes(&winner.worktree) {
7827                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7828                Ok(free) => {
7829                    self.state.event(
7830                        "gate",
7831                        format!(
7832                            "only {free} bytes free ({min_free} required by `[disk] \
7833                             min_free_bytes`); not spending a fix round on a failure the disk \
7834                             may explain"
7835                        ),
7836                    );
7837                    return Ok(GateFix::Stop);
7838                }
7839                Err(e) => {
7840                    self.state.event(
7841                        "gate",
7842                        format!("free disk space could not be measured ({e:#}); no fix round"),
7843                    );
7844                    return Ok(GateFix::Stop);
7845                }
7846            }
7847        }
7848
7849        let attempt = spent + 1;
7850        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7851        let base = self.landing_base();
7852        let fix_prompt = prompt::gate_fix_in(
7853            &self.state.instruction,
7854            &failed,
7855            attempt,
7856            cap,
7857            &self.state.config.graph.language,
7858            &self.state.config.graph.github_language,
7859        );
7860        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7861        let sessions = self.state.config.graph.sessions;
7862        let artifacts = self.merge_repair_artifacts();
7863        self.state.event(
7864            "gate",
7865            format!("gate failed; gate-fix round {attempt} of {cap}"),
7866        );
7867        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7868        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7869        // Persist before invoking the fixer: it may commit and then be
7870        // interrupted before its result (or the following e2e) is saved.
7871        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7872            head: before.clone(),
7873            passed: false,
7874            outcomes: Vec::new(),
7875        });
7876        self.state.save()?;
7877        let (_, seat, out) = self
7878            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7879                prompt: fix_prompt.clone(),
7880                spec,
7881                seat,
7882                cwd: winner.worktree.clone(),
7883                timeout,
7884                allow_write: true,
7885                sessions,
7886                artifacts: artifacts.clone(),
7887                stem: format!("gate-fix-{attempt}"),
7888                handover: None,
7889            })
7890            .await;
7891        // The fixer was never asked: no round is spent (nothing is recorded)
7892        // and a later reentry asks the same agent again.
7893        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7894            return Ok(GateFix::Defer);
7895        }
7896        let mut record = GateFixRecord {
7897            agent: seat.agent.clone(),
7898            failed,
7899            notes: String::new(),
7900            committed: false,
7901            error: None,
7902        };
7903        match out {
7904            AgentOutcome::Ok(o) => {
7905                // A missing report is not a failed fix: the round is judged
7906                // by the tree below, and the report only carries prose.
7907                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7908                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7909                }
7910            }
7911            AgentOutcome::Dropped(_) => {
7912                record.error = Some("the CLI dropped the stream".to_owned());
7913            }
7914            AgentOutcome::Quota(o) => {
7915                self.state.quota.push(QuotaLoss::from_report(
7916                    seat.key.clone(),
7917                    "gate-fix",
7918                    o.quota.as_ref(),
7919                ));
7920                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7921            }
7922            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7923        }
7924        self.state.seats.insert(seat.key.clone(), seat);
7925        let coordination = self.coordinate(&winner.worktree).await?;
7926        if let Ok(r) = git::rescue_commit(
7927            &winner.worktree,
7928            &format!("magi: gate fix {attempt} (uncommitted work)"),
7929        )
7930        .await
7931        {
7932            self.state.note_withheld("gate-fix", &r.withheld);
7933        }
7934        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7935        record.committed = after != before;
7936        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7937        let note = record.error.clone();
7938        self.state.gate_fixes.push(record);
7939        self.state.save()?;
7940        if !changed {
7941            self.state.event(
7942                "gate",
7943                match note {
7944                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7945                    None => format!("gate-fix round {attempt}: the tree did not change"),
7946                },
7947            );
7948            return Ok(GateFix::Stop);
7949        }
7950        self.state.event(
7951            "gate",
7952            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7953        );
7954
7955        drop(coordination);
7956        let wait = self.repository_wait();
7957        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7958    }
7959
7960    // ---------------------------------------------------------------- merge
7961
7962    /// One read-only rewrite by the summary's author; a text the gate still
7963    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7964    /// applies on their word or on silence. `None` means the run is parked on
7965    /// that question and nothing may be pushed or opened yet.
7966    async fn guarded_pr_message(
7967        &mut self,
7968        winner: &Candidate,
7969        facts: Option<&BranchFacts>,
7970        posting: bool,
7971    ) -> Result<Option<PrMessage>> {
7972        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7973        let head = match final_head.as_deref() {
7974            Some(h) => crate::pr_outcome::FinalHead::At(h),
7975            None => crate::pr_outcome::FinalHead::Unknown,
7976        };
7977        // Only a pull request that is about to be posted pays for a summary.
7978        let summarized = if posting {
7979            Some(self.summarize_change(winner, final_head.as_deref()).await)
7980        } else {
7981            None
7982        };
7983        // A busy shared resource is not a permanent cause: publishing an
7984        // "unavailable" description would stick (a later head move never
7985        // edits the body). Stop before anything is pushed or opened; the
7986        // run is `Blocked` and a resume asks the summary again.
7987        if self.state.resource_block.is_some() {
7988            self.state.event(
7989                "pr-summary",
7990                "a shared resource was busy; nothing was published, resume to summarize again",
7991            );
7992            return Ok(None);
7993        }
7994        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7995            anyhow::bail!(
7996                "the final commit moved during public description generation; retry before publishing"
7997            );
7998        }
7999        let note = match (&summarized, &self.state.change_summary) {
8000            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8001            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8002            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8003            (None, _) => ChangeNote::NotAsked,
8004        };
8005        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8006        if !posting {
8007            let identity = crate::scrub::Identity::current();
8008            return Ok(Some(PrMessage {
8009                title: crate::scrub::scrub(&pr.title, &identity),
8010                body: crate::scrub::scrub(&pr.body, &identity),
8011            }));
8012        }
8013        // This very text was already rejected and asked about: no second
8014        // judge call, rewrite or question, just carry the record on.
8015        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8016        if let Some(known) = self
8017            .state
8018            .github_text
8019            .clone()
8020            .filter(|g| g.fingerprint == raw_fp)
8021        {
8022            return self.settle_withheld(winner, &pr, known).await;
8023        }
8024        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8025        let mut violations =
8026            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8027        let identity = crate::scrub::Identity::current();
8028        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8029            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8030            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8031        {
8032            violations.push(crate::github_text::Violation::SensitiveData);
8033        }
8034        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8035        // only a language violation starts one; sensitive data alone goes
8036        // straight to the owner, with the redacted text as the fallback.
8037        let has_language = violations
8038            .iter()
8039            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8040        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8041            self.state.event(
8042                "github-text",
8043                if has_language {
8044                    format!("description rejected: {violations:?}; requesting one rewrite")
8045                } else {
8046                    format!("description rejected: {violations:?}; asking the owner")
8047                },
8048            );
8049            let mut rewritten = false;
8050            let mut rejected_rewrite = None;
8051            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8052            {
8053                let output = text;
8054                if let Some(title) = summary_title(&output) {
8055                    // Judge the redacted candidate that will actually be posted.
8056                    // Sensitive data alone must not discard a useful rewrite.
8057                    let raw_body = summary_without_title(&output);
8058                    let clean_title = crate::scrub::scrub(&title, &identity);
8059                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8060                    let redacted = clean_title != title || body != raw_body;
8061                    let title = clean_title;
8062                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8063                    if !complete {
8064                        self.state.event(
8065                            "github-text",
8066                            "description rewrite omitted required findings",
8067                        );
8068                    }
8069                    if !body.trim().is_empty() {
8070                        body.push_str("\n\n");
8071                        body.push_str(&public_pr_evidence(&self.state, head));
8072                    }
8073                    let marker = format!("magi:run/{}", self.state.id);
8074                    if !body.trim().is_empty() && !body.contains(&marker) {
8075                        body.push_str(&format!("\n\n{marker}"));
8076                    }
8077                    let rewrite = PrMessage {
8078                        title: title.clone(),
8079                        body: body.clone(),
8080                    };
8081                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8082                    let mut failed = crate::github_text::check_for(
8083                        &self.state.config.graph,
8084                        &title,
8085                        &body,
8086                        redecision,
8087                    );
8088                    if (!crate::github_text::shareable(&title)
8089                        || !crate::github_text::shareable(&body))
8090                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8091                    {
8092                        failed.push(crate::github_text::Violation::SensitiveData);
8093                    }
8094                    if complete
8095                        && !title.trim().is_empty()
8096                        && !body.trim().is_empty()
8097                        && !failed.is_empty()
8098                    {
8099                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8100                            title: title.clone(),
8101                            body: body.clone(),
8102                            failed: failed
8103                                .iter()
8104                                .map(|v| crate::github_text::category(*v).to_owned())
8105                                .collect(),
8106                        });
8107                    }
8108                    if complete
8109                        && !title.trim().is_empty()
8110                        && !raw_body.trim().is_empty()
8111                        && failed.is_empty()
8112                    {
8113                        decision = redecision;
8114                        pr = PrMessage { title, body };
8115                        rewritten = true;
8116                        self.state
8117                            .event("github-text", "description rewrite passed");
8118                        if redacted {
8119                            self.state.event(
8120                                "github-text",
8121                                "sensitive data removed from description rewrite",
8122                            );
8123                        }
8124                    }
8125                }
8126            }
8127            if !rewritten {
8128                self.state.event(
8129                    "github-text",
8130                    "description rewrite unavailable or rejected; asking the owner",
8131                );
8132                if let Some(w) =
8133                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8134                {
8135                    // A different text than the one asked about before: that
8136                    // question no longer describes anything.
8137                    self.retire_github_text_question("the withheld text changed");
8138                    let record = crate::run::GithubTextAsk {
8139                        rewrite: rejected_rewrite,
8140                        fingerprint: raw_fp,
8141                        title: w.title,
8142                        body: w.body,
8143                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8144                        question: None,
8145                        asks: 0,
8146                        resolved: false,
8147                        chosen_title: None,
8148                        choice: None,
8149                    };
8150                    return self.settle_withheld(winner, &pr, record).await;
8151                }
8152            }
8153        }
8154        let (title, body) =
8155            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8156        Ok(Some(PrMessage { title, body }))
8157    }
8158
8159    /// Describe the whole change from its final diff in either run mode.
8160    ///
8161    /// The summary is tied to the final commit and to the merge base of the
8162    /// resolved remote base; a record for any other pair is dropped and the
8163    /// agent asked again, so a description never outlives the diff it was
8164    /// written from. The seat is read-only and the prompt tells it to read the
8165    /// diff and the code around it. `[roles] synthesizer` selects the agents
8166    /// (unset: the default pick), each asked at most once, advancing on an
8167    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8168    /// carries the reason it is unavailable; nothing is invented then.
8169    async fn summarize_change(
8170        &mut self,
8171        winner: &Candidate,
8172        head: Option<&str>,
8173    ) -> std::result::Result<(), String> {
8174        let Some(head) = head else {
8175            return Err("the final commit could not be read".to_owned());
8176        };
8177        let remote = self.state.config.merge.remote.clone();
8178        let tracking = format!("{remote}/{}", self.state.base_branch);
8179        if !git::rev_exists(&winner.worktree, &tracking).await {
8180            return Err(format!("the remote base `{tracking}` could not be read"));
8181        }
8182        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8183            Ok(b) if !b.is_empty() => b,
8184            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8185        };
8186        let evidence = crate::pr_outcome::summary_evidence(
8187            &self.state,
8188            crate::pr_outcome::FinalHead::At(head),
8189        );
8190        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8191        if self
8192            .state
8193            .change_summary
8194            .as_ref()
8195            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8196        {
8197            return Ok(());
8198        }
8199        if self.state.change_summary.take().is_some() {
8200            self.state.event(
8201                "pr-summary",
8202                "the final diff or evidence changed; the earlier change summary was discarded",
8203            );
8204        }
8205        // Save the source snapshot even if no generation agent is available.
8206        if let Err(e) = write_artifact(
8207            &self.state,
8208            &format!("public-description-{fingerprint}.md"),
8209            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8210        ) {
8211            tracing::warn!("could not save public description evidence: {e:#}");
8212        }
8213        let chain = match agent::pick_chain(
8214            &self.state.config.agents,
8215            self.state.config.roles.synthesizer.as_ref(),
8216            &agent::installed,
8217            "synthesizer",
8218        ) {
8219            Ok(chain) => chain,
8220            Err(e) => {
8221                tracing::warn!("no agent for the change summary: {e:#}");
8222                self.state.event("pr-summary", "no summary agent available");
8223                return Err("no agent was available to summarize it".to_owned());
8224            }
8225        };
8226        let subjects = git::subjects(&winner.worktree, &base, head)
8227            .await
8228            .unwrap_or_default();
8229        let stat = git::diff_stat(&winner.worktree, &base, head)
8230            .await
8231            .unwrap_or_default();
8232        let stat: String = stat.chars().take(6_000).collect();
8233        let mut prompt = crate::prompt::change_summary_in(
8234            &base,
8235            head,
8236            &subjects,
8237            stat.trim(),
8238            &self.state.config.graph.github_language,
8239        );
8240        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8241        let sessions = self.state.config.graph.sessions;
8242        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8243        let prompts = self.state.config.prompts.clone();
8244        let cache = self.state.config.cache_dir();
8245        let run = self.state.id.clone();
8246        for (i, spec) in chain.into_iter().enumerate() {
8247            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8248            let github_spec = match agent::for_github_text(
8249                &spec,
8250                &self.state.config.graph.github_language,
8251                &winner.worktree,
8252            ) {
8253                Ok(spec) => spec,
8254                Err(e) => {
8255                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8256                    continue;
8257                }
8258            };
8259            let job = SeatJob {
8260                spec: github_spec,
8261                seat,
8262                cwd: winner.worktree.clone(),
8263                prompt: prompt.clone(),
8264                timeout,
8265                allow_write: false,
8266                sessions,
8267                artifacts: agent::artifacts_dir(&self.state.dir()),
8268                stem: if i == 0 {
8269                    "change-summary".to_owned()
8270                } else {
8271                    format!("change-summary-{}", spec.id)
8272                },
8273                handover: None,
8274            };
8275            let ctx = WaveCtx {
8276                run: &run,
8277                node: "pr-summary",
8278                prompts: &prompts,
8279                cache: cache.as_deref(),
8280                round: None,
8281                carry_seats: false,
8282                pauses: Some((&self.pause, &self.interrupt)),
8283            };
8284            let (seat, outcome) =
8285                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8286            self.state.seats.insert(seat.key.clone(), seat);
8287            let text = match outcome {
8288                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8289                AgentOutcome::ResourceBlocked(why) => {
8290                    // No agent was asked: the run stops resumably before
8291                    // anything is published (see `guarded_pr_message`).
8292                    if self.state.resource_block.is_none() {
8293                        self.state.resource_block = Some(crate::run::ResourceBlock {
8294                            cause: crate::run::ResourceBlockCause::Other,
8295                            node: "pr-summary".to_owned(),
8296                            kind: "shared resource".to_owned(),
8297                            detail: why,
8298                            at: Timestamp::now(),
8299                        });
8300                    }
8301                    return Err("a shared resource was busy".to_owned());
8302                }
8303                _ => continue,
8304            };
8305            let blind = &self.state.config.blind;
8306            let Some(title) = summary_title(&text) else {
8307                continue;
8308            };
8309            let body: String = summary_without_title(&text)
8310                .lines()
8311                .filter(|l| !l.contains("magi:run/"))
8312                .collect::<Vec<_>>()
8313                .join("\n");
8314            let body = blind::sanitize_prose(body.trim(), blind);
8315            let title = blind::sanitize_prose(&title, blind);
8316            if body.trim().is_empty() || title.trim().is_empty() {
8317                continue;
8318            }
8319            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8320                continue;
8321            }
8322            // The branch must still be the one the agent was shown.
8323            if git::rev_parse(&winner.worktree, "HEAD")
8324                .await
8325                .ok()
8326                .as_deref()
8327                != Some(head)
8328            {
8329                return Err("the branch moved while the summary was written".to_owned());
8330            }
8331            if git::merge_base(&winner.worktree, &tracking, head)
8332                .await
8333                .ok()
8334                .as_deref()
8335                != Some(base.as_str())
8336                || crate::pr_outcome::summary_evidence(
8337                    &self.state,
8338                    crate::pr_outcome::FinalHead::At(head),
8339                ) != evidence
8340            {
8341                return Err("the summary evidence changed while it was written".to_owned());
8342            }
8343            self.state.change_summary = Some(crate::run::ChangeSummary {
8344                head: head.to_owned(),
8345                base,
8346                evidence: fingerprint,
8347                title,
8348                body,
8349            });
8350            self.state.event(
8351                "pr-summary",
8352                format!("change summary written by {} for the final diff", spec.id),
8353            );
8354            if let Err(e) = self.state.save() {
8355                tracing::warn!("could not save the change summary: {e:#}");
8356            }
8357            return Ok(());
8358        }
8359        self.state
8360            .event("pr-summary", "no agent produced a change summary");
8361        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8362    }
8363
8364    /// One read-only rewrite request for a withheld description. The
8365    /// implementer's own conversation is tried first when it exists and can be
8366    /// resumed; then the `[roles] language_judge` chain (or the default pick)
8367    /// writes it in a separate conversation, skipping the implementer's agent.
8368    /// The prompt carries the whole text, so no agent needs any memory. Each
8369    /// agent is asked at most once and the chain advances on an error, quota,
8370    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
8371    /// that has run out of quota must not end the rewrite. Nothing is recorded
8372    /// as a handover.
8373    async fn rewrite_pr_text(
8374        &mut self,
8375        winner: &Candidate,
8376        pr: &PrMessage,
8377        violations: &[crate::github_text::Violation],
8378    ) -> Option<String> {
8379        let prompt = format!(
8380            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8381            pr.title, pr.body
8382        );
8383        let sessions = self.state.config.graph.sessions;
8384        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8385        // (spec, the resumed implementer's seat). A fallback seat is created
8386        // only when its turn comes, so an unused fallback leaves no record.
8387        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8388        let key = format!("impl-{}", winner.label);
8389        if let Some(spec) = self
8390            .state
8391            .config
8392            .agents
8393            .iter()
8394            .find(|a| a.id == winner.agent)
8395            .cloned()
8396            && let Some(existing) = self.state.seats.get(&key)
8397            && existing.agent == spec.id
8398            && agent::has_session(spec.kind, existing, sessions)
8399        {
8400            attempts.push((spec, Some(existing.clone())));
8401        }
8402        match agent::pick_chain(
8403            &self.state.config.agents,
8404            self.state.config.roles.language_judge.as_ref(),
8405            &agent::installed,
8406            "language_judge",
8407        ) {
8408            Ok(chain) => {
8409                for spec in chain {
8410                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8411                        continue;
8412                    }
8413                    attempts.push((spec, None));
8414                }
8415            }
8416            Err(e) if attempts.is_empty() => {
8417                tracing::warn!("no agent for the description rewrite: {e:#}");
8418                self.state
8419                    .event("github-text", "no rewrite agent available");
8420                return None;
8421            }
8422            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8423        }
8424        let prompts = self.state.config.prompts.clone();
8425        let cache = self.state.config.cache_dir();
8426        let run = self.state.id.clone();
8427        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8428            let resumed = resumed_seat.is_some();
8429            let seat = resumed_seat.unwrap_or_else(|| {
8430                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8431            });
8432            let github_spec = match agent::for_github_text(
8433                &spec,
8434                &self.state.config.graph.github_language,
8435                &winner.worktree,
8436            ) {
8437                Ok(spec) => spec,
8438                Err(e) => {
8439                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8440                    continue;
8441                }
8442            };
8443            let job = SeatJob {
8444                spec: github_spec,
8445                seat,
8446                cwd: winner.worktree.clone(),
8447                prompt: prompt.clone(),
8448                timeout: retry_budget(timeout_full, resumed),
8449                allow_write: false,
8450                sessions,
8451                artifacts: agent::artifacts_dir(&self.state.dir()),
8452                stem: if i == 0 {
8453                    "github-text-rewrite".to_owned()
8454                } else {
8455                    format!("github-text-rewrite-{}", spec.id)
8456                },
8457                handover: None,
8458            };
8459            let ctx = WaveCtx {
8460                run: &run,
8461                node: "github-text",
8462                prompts: &prompts,
8463                cache: cache.as_deref(),
8464                round: None,
8465                carry_seats: false,
8466                pauses: Some((&self.pause, &self.interrupt)),
8467            };
8468            let (seat, outcome) =
8469                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8470            self.state.seats.insert(seat.key.clone(), seat);
8471            match outcome {
8472                AgentOutcome::Ok(o)
8473                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8474                {
8475                    return Some(o.text);
8476                }
8477                AgentOutcome::ResourceBlocked(_) => return None,
8478                _ => {}
8479            }
8480        }
8481        None
8482    }
8483
8484    /// Abandon the standing posting-gate question, if any.
8485    fn retire_github_text_question(&mut self, why: &str) {
8486        let Some(id) = self
8487            .state
8488            .github_text
8489            .as_ref()
8490            .and_then(|g| g.question.clone())
8491        else {
8492            return;
8493        };
8494        let _ = ask::Questions::open().update(&id, |q| {
8495            q.abandon(why);
8496            Ok(())
8497        });
8498    }
8499
8500    /// Carry a withheld text to its end: file the question, park while it is
8501    /// open, and on an answer, silence or timeout pick the text to post. The
8502    /// decision is saved before it is used, so it applies once.
8503    async fn settle_withheld(
8504        &mut self,
8505        winner: &Candidate,
8506        pr: &PrMessage,
8507        mut g: crate::run::GithubTextAsk,
8508    ) -> Result<Option<PrMessage>> {
8509        use crate::github_text as gt;
8510        let store = ask::Questions::open();
8511        let timeout = self.state.config.graph.answer_timeout;
8512        if !g.resolved {
8513            let mut standing = g
8514                .question
8515                .as_ref()
8516                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8517            if let Some(q) = standing.as_ref()
8518                && q.status.open()
8519                && gt::expired(q, timeout)
8520            {
8521                let why = format!("no answer within {}s of asking", timeout.max(1));
8522                standing = store
8523                    .update(&q.id, |q| {
8524                        q.abandon(&why);
8525                        Ok(())
8526                    })
8527                    .ok()
8528                    .map(|(q, ())| q);
8529            }
8530            let mut ask_again: Option<Option<Vec<String>>> = None;
8531            match standing {
8532                // Recorded but never written (a stop between the two saves),
8533                // or removed: ask again without spending the retry.
8534                None if g.asks <= 1 => {
8535                    g.asks = 0;
8536                    ask_again = Some(None);
8537                }
8538                None => {
8539                    g.resolved = true;
8540                    g.chosen_title = None;
8541                    g.choice = Some(GithubTextChoice::Fallback);
8542                }
8543                Some(q) if q.status.open() => {
8544                    self.park_on_github_text(&g)?;
8545                    return Ok(None);
8546                }
8547                Some(q) => match gt::read_reply(&q) {
8548                    gt::Reply::Fallback => {
8549                        g.resolved = true;
8550                        g.chosen_title = None;
8551                        g.choice = Some(GithubTextChoice::Fallback);
8552                    }
8553                    gt::Reply::Original => {
8554                        g.resolved = true;
8555                        g.chosen_title = None;
8556                        g.choice = Some(GithubTextChoice::Original);
8557                    }
8558                    gt::Reply::Rewrite => {
8559                        g.resolved = true;
8560                        g.chosen_title = None;
8561                        g.choice = Some(if g.rewrite.is_some() {
8562                            GithubTextChoice::Rewrite
8563                        } else {
8564                            GithubTextChoice::Fallback
8565                        });
8566                    }
8567                    gt::Reply::Title(text) => {
8568                        let first = text
8569                            .lines()
8570                            .map(str::trim)
8571                            .find(|l| !l.is_empty())
8572                            .unwrap_or("");
8573                        let decision = if first.is_empty() {
8574                            None
8575                        } else {
8576                            let candidate = PrMessage {
8577                                title: first.to_owned(),
8578                                body: String::new(),
8579                            };
8580                            self.judge_pr_language(&winner.worktree, &candidate).await
8581                        };
8582                        match gt::vet_title(
8583                            &text,
8584                            decision,
8585                            self.state.config.graph.github_english(),
8586                        ) {
8587                            Ok(title) => {
8588                                g.resolved = true;
8589                                g.chosen_title = Some(title);
8590                                g.choice = Some(GithubTextChoice::MyTitle);
8591                            }
8592                            Err(cats) if g.asks < 2 => {
8593                                ask_again = Some(Some(
8594                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8595                                ))
8596                            }
8597                            Err(_) => {
8598                                g.resolved = true;
8599                                g.chosen_title = None;
8600                                g.choice = Some(GithubTextChoice::Fallback);
8601                            }
8602                        }
8603                    }
8604                },
8605            }
8606            if let Some(rejected) = ask_again {
8607                // A retry (after a rejected replacement title) is still about
8608                // the original text: the same ORIGINAL / FALLBACK comparison.
8609                let w = gt::Withheld {
8610                    title: g.title,
8611                    body: g.body,
8612                    categories: g.categories.clone(),
8613                };
8614                // Named by the original text's fingerprint alone, so a resume
8615                // rewrites the same file and the detail stays identical.
8616                let artifact = match crate::run::write_artifact(
8617                    &self.state,
8618                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
8619                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8620                ) {
8621                    Ok(p) => Some(p),
8622                    Err(e) => {
8623                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
8624                        None
8625                    }
8626                };
8627                let (detail, comparison) = gt::question_content(
8628                    &w,
8629                    &pr.title,
8630                    &pr.body,
8631                    &self.state.id,
8632                    rejected.as_deref(),
8633                    artifact.as_deref(),
8634                    g.rewrite.as_ref(),
8635                );
8636                let mut q = ask::Question::new(
8637                    self.state.id.clone(),
8638                    gt::ASK_NODE.to_owned(),
8639                    gt::ASK_SEAT.to_owned(),
8640                    gt::question_summary(&self.state.config.graph.language, &w),
8641                    detail,
8642                    gt::question_choices(&w, g.rewrite.as_ref()),
8643                );
8644                q.posting_comparison = Some(comparison);
8645                q.answer_timeout = timeout;
8646                // Recorded before it is written: a stop in between leaves a
8647                // record naming a question the resume then files again.
8648                g.question = Some(q.id.clone());
8649                g.asks += 1;
8650                self.state.github_text = Some(g.clone());
8651                self.state.event(
8652                    "github-text",
8653                    format!("asking the owner about the withheld text ({})", q.short()),
8654                );
8655                self.state.save()?;
8656                store
8657                    .put_executable(&mut q)
8658                    .context("file the posting-gate question")?;
8659                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8660                    tracing::warn!(
8661                        "could not notify about posting-gate question {}: {e:#}",
8662                        q.short()
8663                    );
8664                }
8665                self.park_on_github_text(&g)?;
8666                return Ok(None);
8667            }
8668        }
8669        let resolved = g.chosen_title.clone();
8670        let original = g.choice == Some(GithubTextChoice::Original);
8671        // Field names only: the event log never records the text.
8672        self.state.event(
8673            "github-text",
8674            if original {
8675                format!(
8676                    "posting the original text for: {}",
8677                    match (g.title, g.body) {
8678                        (true, true) => "title, description",
8679                        (true, false) => "title",
8680                        _ => "description",
8681                    }
8682                )
8683            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8684                "posting the saved rewrite title and description".to_owned()
8685            } else if resolved.is_some() {
8686                "posting the owner's replacement title".to_owned()
8687            } else {
8688                "posting the neutral text for the withheld fields".to_owned()
8689            },
8690        );
8691        self.state.github_text = Some(g.clone());
8692        self.state.save()?;
8693        if g.choice == Some(GithubTextChoice::Rewrite)
8694            && let Some(r) = g.rewrite.as_ref()
8695        {
8696            return Ok(Some(PrMessage {
8697                title: r.title.clone(),
8698                body: r.body.clone(),
8699            }));
8700        }
8701        let w = gt::Withheld {
8702            title: g.title,
8703            body: g.body,
8704            categories: g.categories.clone(),
8705        };
8706        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8707        if let Some(t) = resolved {
8708            title = t;
8709        }
8710        // Every withheld field is now neutral or vetted English.
8711        let english = gt::LanguageDecision {
8712            title_english: true,
8713            body_english: true,
8714            body_complete: true,
8715        };
8716        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8717        if original {
8718            // The owner's explicit choice overrides the gate for the fields it
8719            // was made for; the others keep what the gate produced.
8720            if g.title {
8721                title = pr.title.clone();
8722            }
8723            if g.body {
8724                body = pr.body.clone();
8725            }
8726        }
8727        Ok(Some(PrMessage { title, body }))
8728    }
8729
8730    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8731        self.state.github_text = Some(g.clone());
8732        self.state.end_pass_parked(ParkCause::Question);
8733        self.state.parked = true;
8734        self.state.event(
8735            "github-text",
8736            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8737        );
8738        self.state.save()
8739    }
8740
8741    /// Ask `[roles] language_judge` about this text and record which source
8742    /// decided; the text itself is never recorded.
8743    async fn judge_pr_language(
8744        &mut self,
8745        cwd: &Path,
8746        pr: &PrMessage,
8747    ) -> Option<crate::github_text::LanguageDecision> {
8748        self.state.config.roles.language_judge.as_ref()?;
8749        if !self.state.config.graph.language_gate() {
8750            return None;
8751        }
8752        let decision =
8753            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8754        self.state.event(
8755            "github-text",
8756            if decision.is_some() {
8757                "language decided by the language judge"
8758            } else {
8759                "language judge unavailable; using built-in heuristics"
8760            },
8761        );
8762        decision
8763    }
8764
8765    async fn merge(&mut self) -> Result<()> {
8766        if self.resource_halt()? || self.hold_base_sync().await? {
8767            return Ok(());
8768        }
8769        // Same reasoning as `gate`: ask the review and gate records directly
8770        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8771        // skip can rewrite on reentry to something that no longer says
8772        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8773        // so a hand-off (open findings, green verification) reaches merge
8774        // exactly like a genuinely clean round does.
8775        //
8776        // A run resumed mid-`land` never reaches here at all: `execute`
8777        // recognises `RunStatus::Landing` before it even calls `prep`, and
8778        // routes straight to `run_land` instead. That has to happen a level
8779        // up from this function, not with a check in here, because
8780        // `review_loop`'s own status recomputation (see its doc) runs
8781        // *before* `merge` on every reentry and would otherwise overwrite
8782        // the `Landing` marker with `Gating` before this node ever saw it.
8783        if self
8784            .state
8785            .base_sync
8786            .as_ref()
8787            .is_some_and(|s| s.conflict.is_some())
8788            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8789                != Some(RunStatus::Gating)
8790            // `gate_ran == false` is not "passed" - `gate` leaves it false
8791            // both before it has ever run and when its last attempt was
8792            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8793            // permission to merge on nothing but the review record. Only a
8794            // gate that actually ran may proceed; an intentional local
8795            // review can also finish with verification explicitly not configured.
8796            // `RunState::gate_status` computes that distinction.
8797            || !self.state.gate_allows_progress()
8798        {
8799            return Ok(());
8800        }
8801        // This node's own record, not `status`: `status == Ready` is not
8802        // unique to the harmless `MergeMode::None` path this line was
8803        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8804        // run's PR was closed without merging — and on that run `mode` is
8805        // still `Pr`, so a reentry that fell through here would push and
8806        // open a second pull request. `self.state.merge` is set exactly once
8807        // this node (or `land`) has already produced a verdict, under every
8808        // mode, which is what "already done" actually means here.
8809        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8810            return Ok(());
8811        }
8812        let Some(winner) = self.state.winner().cloned() else {
8813            return Ok(());
8814        };
8815        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8816        if self.state.gate_head.as_deref() != Some(head.as_str()) {
8817            self.state.status = RunStatus::Blocked;
8818            self.state.event(
8819                "gate",
8820                "winner changed after gate; fresh verification required",
8821            );
8822            self.state.save()?;
8823            return Ok(());
8824        }
8825        let wait = self.repository_wait();
8826        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8827            return Ok(());
8828        }
8829        let repo = self.state.repo.clone();
8830        let base = self.state.base_branch.clone();
8831        let mode = self.state.config.merge.mode;
8832        let style = self.state.config.merge.style;
8833        let facts = if is_review_run(&self.state) {
8834            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8835            let start = review_base(
8836                &repo,
8837                &self.state.config.merge.remote,
8838                &base,
8839                &self.state.base_commit,
8840                &winner.branch,
8841            )
8842            .await;
8843            branch_facts(&repo, &start, &winner.branch).await
8844        } else {
8845            None
8846        };
8847        // `None` when the base could not be freshly read: then an adopted
8848        // pull request's title is left alone.
8849        let leaked = if is_review_run(&self.state) {
8850            leaked_subjects(&self.state, &winner.branch).await
8851        } else {
8852            Some(Vec::new())
8853        };
8854        // `None`: parked on the owner's word about a withheld title or body.
8855        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8856        // the resume arrives here again.
8857        let Some(pr) = self
8858            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8859            .await?
8860        else {
8861            return Ok(());
8862        };
8863        let message = pr.commit_message();
8864
8865        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
8866        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
8867            return Ok(());
8868        }
8869        let outcome = match mode {
8870            MergeMode::None => MergeOutcome {
8871                mode,
8872                ok: true,
8873                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8874                empty: false,
8875            },
8876            MergeMode::Pr | MergeMode::Local
8877                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8878            {
8879                MergeOutcome {
8880                    mode,
8881                    ok: false,
8882                    detail: empty_candidate_detail(&self.state, &base),
8883                    empty: true,
8884                }
8885            }
8886            MergeMode::Local => {
8887                let on = git::current_branch(&repo).await?;
8888                if on.as_deref() != Some(base.as_str()) {
8889                    MergeOutcome {
8890                        mode,
8891                        ok: false,
8892                        detail: format!(
8893                            "{} has {} checked out, not the base branch {base}",
8894                            repo.display(),
8895                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8896                        ),
8897                        empty: false,
8898                    }
8899                } else if !git::is_clean(&repo).await? {
8900                    MergeOutcome {
8901                        mode,
8902                        ok: false,
8903                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8904                        empty: false,
8905                    }
8906                } else {
8907                    let out = match style {
8908                        MergeStyle::Merge => {
8909                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8910                        }
8911                        MergeStyle::Squash => {
8912                            git::merge_squash(&repo, &winner.branch, &message).await?
8913                        }
8914                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8915                    };
8916                    MergeOutcome {
8917                        mode,
8918                        ok: out.ok(),
8919                        detail: if out.ok() { out.stdout } else { out.stderr },
8920                        empty: false,
8921                    }
8922                }
8923            }
8924            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8925                self.state.pr_publication_pending = true;
8926                self.state.save()?;
8927                let out =
8928                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8929                self.state.pr_publication_pending = out.is_err();
8930                MergeOutcome {
8931                    mode,
8932                    ok: out.is_ok(),
8933                    detail: match out {
8934                        Ok(url) => url,
8935                        Err(e) => format!("{e:#}"),
8936                    },
8937                    empty: false,
8938                }
8939            }
8940            MergeMode::Pr => {
8941                let remote = self.state.config.merge.remote.clone();
8942                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8943                let pushed = if self.state.foreign_checkout.is_some() {
8944                    let refspec = format!("{delivery_head}:refs/heads/{public}");
8945                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
8946                } else {
8947                    git::push(&winner.worktree, &remote, &winner.branch).await?
8948                };
8949                if !pushed.ok() {
8950                    MergeOutcome {
8951                        mode,
8952                        ok: false,
8953                        detail: pushed.stderr,
8954                        empty: false,
8955                    }
8956                } else {
8957                    if self.state.foreign_checkout.is_some() {
8958                        self.state.pr_pushed_head = Some(delivery_head.clone());
8959                    }
8960                    // A retry or resume of a run whose branch already has an
8961                    // open pull request adopts it rather than failing on a
8962                    // duplicate. Only this winner branch into this base:
8963                    // `branch_for` derives the name from the run id, so a
8964                    // different run's pull request never matches.
8965                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
8966                    let out = match pr_merge_plan(found) {
8967                        PrPlan::Create => {
8968                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
8969                                .await
8970                        }
8971                        PrPlan::Adopt { url, title } => {
8972                            self.state
8973                                .event("merge", format!("Pr: adopted open pull request {url}"));
8974                            if self.state.github_text.as_ref().is_some_and(|g| {
8975                                g.resolved
8976                                    && g.choice == Some(GithubTextChoice::Rewrite)
8977                                    && g.rewrite.is_some()
8978                            }) {
8979                                // The owner chose the entire pair, even if the
8980                                // existing title already matches or was edited.
8981                                land::set_pr_text_verbatim(
8982                                    &winner.worktree,
8983                                    &url,
8984                                    &pr.title,
8985                                    &pr.body,
8986                                )
8987                                .await
8988                                .map(|()| url)
8989                            } else {
8990                                if title != pr.title
8991                                    && (!is_review_run(&self.state)
8992                                        || leaked
8993                                            .as_deref()
8994                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8995                                    && let Err(e) =
8996                                        if self.state.github_text.as_ref().is_some_and(|g| {
8997                                            g.resolved
8998                                                && g.title
8999                                                && g.choice == Some(GithubTextChoice::Original)
9000                                        }) {
9001                                            // The owner chose to post this title unmodified.
9002                                            land::set_pr_title_verbatim(
9003                                                &winner.worktree,
9004                                                &url,
9005                                                &pr.title,
9006                                            )
9007                                            .await
9008                                        } else {
9009                                            land::set_pr_title(
9010                                                &mut self.state,
9011                                                &winner.worktree,
9012                                                &url,
9013                                                &pr.title,
9014                                            )
9015                                            .await
9016                                        }
9017                                {
9018                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9019                                    self.state
9020                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9021                                }
9022                                Ok(url)
9023                            }
9024                        }
9025                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9026                    };
9027                    match out {
9028                        Ok(url) => MergeOutcome {
9029                            mode,
9030                            ok: true,
9031                            detail: url,
9032                            empty: false,
9033                        },
9034                        Err(e) => MergeOutcome {
9035                            mode,
9036                            ok: false,
9037                            detail: e.to_string(),
9038                            empty: false,
9039                        },
9040                    }
9041                }
9042            }
9043        };
9044
9045        self.state.status = match (mode, outcome.ok) {
9046            (MergeMode::None, _) => RunStatus::Ready,
9047            (_, true) if outcome.empty => RunStatus::Ready,
9048            (MergeMode::Pr, true) => RunStatus::Ready,
9049            (_, true) => RunStatus::Merged,
9050            (_, false) => RunStatus::Blocked,
9051        };
9052        self.state.event(
9053            "merge",
9054            format!(
9055                "{:?}: {}",
9056                mode,
9057                outcome.detail.lines().next().unwrap_or("")
9058            ),
9059        );
9060        self.state.merge = Some(outcome);
9061        self.state.save()?;
9062
9063        // The PR is open and the run would historically stop here, leaving the
9064        // operator to watch checks, feed review comments back to a fixer, and
9065        // merge. That was done by hand six times in one session before this
9066        // existed. Opt-in, because merging is the one irreversible thing magi
9067        // can do to a repository.
9068        if self.state.config.graph.land
9069            && mode == MergeMode::Pr
9070            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9071        {
9072            self.run_land().await?;
9073        }
9074        // `run_land` may have left `status` at `Landing` - still waiting on
9075        // CI or the owner's approval, not actually settled - so this has to
9076        // read whatever `status` ended up as here, not the `Merged` this
9077        // function set a few lines up.
9078        self.settle_questions();
9079        Ok(())
9080    }
9081
9082    /// Enter `land`.
9083    ///
9084    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9085    /// a resumed run's re-entry. `land::land` itself is what serialises the
9086    /// two git-mutating moments inside the loop — the rebase push and
9087    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9088    /// to hold a lock across the whole call, and doing so would serialise
9089    /// this run's CI wait against a *different* run's land-approval resume
9090    /// in the same repository, which is exactly the "must not wait on
9091    /// another task" property the daemon's slot-freeing exists to give.
9092    async fn run_land(&mut self) -> Result<()> {
9093        crate::pr_head::ensure_owner(&self.state)?;
9094        let url = self
9095            .state
9096            .pr
9097            .as_ref()
9098            .map(|pr| pr.url.clone())
9099            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9100            .unwrap_or_default();
9101        let url = url.lines().next().unwrap_or("").trim().to_owned();
9102        if !url.starts_with("http") {
9103            return Ok(());
9104        }
9105        // A land failure is not a lost run: the work is on a branch and the
9106        // pull request is open, which is exactly where a human takes over.
9107        loop {
9108            match land::land(&mut self.state, &url).await {
9109                Ok(_)
9110                    if self
9111                        .state
9112                        .merge_repairs
9113                        .last()
9114                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9115                {
9116                    if self.repair_merge_review().await? {
9117                        continue;
9118                    }
9119                }
9120                Ok(_) if self.state.resource_block.is_some() => {
9121                    return self.stop_for_resource();
9122                }
9123                Ok(pr) if self.state.parked => {
9124                    // `land` already saved the parked marker; nothing here
9125                    // overrides `status` back to a terminal value while an
9126                    // approval is still outstanding.
9127                    let _ = pr;
9128                }
9129                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9130                    self.state.status = RunStatus::Superseded;
9131                    self.state.save()?;
9132                }
9133                Ok(pr) => {
9134                    self.state.status = match pr.state {
9135                        land::PrLifecycle::Merged => RunStatus::Merged,
9136                        _ => RunStatus::Blocked,
9137                    };
9138                    // Downstream of a confirmed merge only - see
9139                    // `bump::should_release_bump`'s own doc for why this one
9140                    // check covers all three of `land`'s success paths.
9141                    // Best-effort: the run already landed, so a failure here
9142                    // (the decision call, `gh`, `cargo`) is recorded and never
9143                    // turns a landed run into a failed one.
9144                    if bump::should_release_bump(self.state.status)
9145                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9146                    {
9147                        // The event is the run's own record. Not-eligible cases
9148                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9149                        // `Err` is a bump that was tried and failed:
9150                        // `after_merge` itself raises the operator notice for
9151                        // that, whether or not a release PR exists yet.
9152                        self.state
9153                            .event("bump", format!("release bump skipped: {e:#}"));
9154                    }
9155                    // Independent of the bump, and best-effort in the same way:
9156                    // findings the merge left open become follow-up tasks.
9157                    if self.state.status == RunStatus::Merged {
9158                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9159                    }
9160                    self.state.save()?;
9161                }
9162                Err(e) => {
9163                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9164                        return Err(e);
9165                    }
9166                    self.state.status = RunStatus::Blocked;
9167                    self.state.event("land", format!("gave up: {e}"));
9168                    self.state.save()?;
9169                }
9170            }
9171            break;
9172        }
9173        Ok(())
9174    }
9175
9176    /// One repair cycle stays under the existing driver/task claim. An
9177    /// interrupted fixer is rolled back and held rather than run twice.
9178    async fn repair_merge_review(&mut self) -> Result<bool> {
9179        use crate::run::MergeRepairPhase as Phase;
9180        crate::pr_head::ensure_owner(&self.state)?;
9181        let i = self.state.merge_repairs.len() - 1;
9182        let request = self.state.merge_repairs[i].clone();
9183        if request.phase == Phase::Fixing {
9184            let restored = match self.state.winner().cloned() {
9185                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9186                None => Err(anyhow::anyhow!("repair has no winning branch")),
9187            };
9188            let why = match restored {
9189                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9190                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9191                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9192            };
9193            self.hold_merge_repair(i, why)?;
9194            return Ok(false);
9195        }
9196        // A completed repair gate is a publication checkpoint. On reentry,
9197        // reject external changes before the ordinary gate path can replace
9198        // that evidence by verifying a new head. Requested repairs still
9199        // start their own review generation below.
9200        if request.phase == Phase::Reviewing && self.state.gate_ran {
9201            let winner = self
9202                .state
9203                .winner()
9204                .context("repair has no winning branch")?;
9205            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9206            if self.state.gate_head.as_deref() != Some(head.as_str())
9207                || !git::is_clean(&winner.worktree).await?
9208            {
9209                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9210                return Ok(false);
9211            }
9212        }
9213        let failure = if i >= self.state.config.graph.land_rounds
9214            || self.state.config.graph.review_rounds == 0
9215        {
9216            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9217        } else {
9218            None
9219        };
9220        if let Some(why) = failure {
9221            self.hold_merge_repair(i, why)?;
9222            return Ok(false);
9223        }
9224        if request.phase == Phase::Requested {
9225            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9226                Ok(pr) => pr,
9227                Err(e) => {
9228                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9229                    return Ok(false);
9230                }
9231            };
9232            self.state.merge_repairs[i].phase = Phase::Fixing;
9233            self.state.land_approval = None;
9234            self.state.land_armed_head = None;
9235            self.state.save()?;
9236            // Serialize the exact records, including finding ids/details,
9237            // reconsideration reasons, final votes and verification output.
9238            let evidence = serde_json::to_string_pretty(&request)?;
9239            let reason = format!(
9240                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9241            );
9242            let budget = self.state.config.graph.land_rounds;
9243            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9244            match result {
9245                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9246                Ok(land::Fixed::Deferred) => {
9247                    // An unavailable fixer did not spend this repair request.
9248                    // Restore partial fallback work before retrying it, as for
9249                    // an interrupted repair, without closing the request.
9250                    let worktree = self
9251                        .state
9252                        .winner()
9253                        .context("repair has no winning branch")?
9254                        .worktree
9255                        .clone();
9256                    self.graft_restore(&worktree, &request.head).await?;
9257                    self.state.merge_repairs[i].phase = Phase::Requested;
9258                    self.stop_for_resource()?;
9259                    return Ok(false);
9260                }
9261                Ok(land::Fixed::Failed(why)) => {
9262                    self.hold_merge_repair(
9263                        i,
9264                        format!(
9265                            "repair failed: {why}; inspect the branch and run an explicit review"
9266                        ),
9267                    )?;
9268                    return Ok(false);
9269                }
9270                Err(e) => {
9271                    self.hold_merge_repair(
9272                        i,
9273                        format!(
9274                            "repair failed: {e:#}; inspect the branch and run an explicit review"
9275                        ),
9276                    )?;
9277                    return Ok(false);
9278                }
9279            }
9280            // Even no change gets a real new reviewer wave. The previous
9281            // evidence remains in the request; a former clean/concluded round
9282            // cannot short-circuit the new review or gate.
9283            self.begin_merge_repair_review(i)?;
9284        }
9285        self.review_loop().await?;
9286        self.gate().await?;
9287        if self.resource_halt()? || self.state.parked {
9288            return Ok(false);
9289        }
9290        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9291            != Some(RunStatus::Gating)
9292            || !self.state.gate_allows_progress()
9293        {
9294            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9295            return Ok(false);
9296        }
9297        // Review and gate may have added commits after the initial repair.
9298        // Publish exactly their final head to the existing PR, with a lease.
9299        let winner = self
9300            .state
9301            .winner()
9302            .cloned()
9303            .context("repair has no winning branch")?;
9304        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9305        if self.state.gate_head.as_deref() != Some(after.as_str())
9306            || !git::is_clean(&winner.worktree).await?
9307        {
9308            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9309            return Ok(false);
9310        }
9311        let expected = self
9312            .state
9313            .pr_pushed_head
9314            .as_deref()
9315            .unwrap_or(&request.head)
9316            .to_owned();
9317        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9318        // Push the checked commit itself: a local ref moving after the check
9319        // must not change which commit the forge receives.
9320        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9321            return Ok(false);
9322        }
9323        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9324        let refspec = format!("{after}:refs/heads/{public}");
9325        let push = git::git_raw(
9326            &winner.worktree,
9327            &["push", &lease, &self.state.config.merge.remote, &refspec],
9328        )
9329        .await;
9330        match push {
9331            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9332            Ok(out) => {
9333                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9334                return Ok(false);
9335            }
9336            Err(e) => {
9337                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9338                return Ok(false);
9339            }
9340        }
9341        self.state.merge_repairs[i].phase = Phase::Complete;
9342        self.state.status = RunStatus::Landing;
9343        self.state.event(
9344            "repair",
9345            "new review and gate completed; fresh owner merge decision required",
9346        );
9347        self.state.save()?;
9348        Ok(true)
9349    }
9350
9351    fn merge_repair_artifacts(&self) -> PathBuf {
9352        let artifacts = agent::artifacts_dir(&self.state.dir());
9353        match self
9354            .state
9355            .merge_repairs
9356            .last()
9357            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9358        {
9359            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9360            None => artifacts,
9361        }
9362    }
9363
9364    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9365        if let Some(e2e) = self.state.head_e2e.take() {
9366            self.state
9367                .verification_history
9368                .push(crate::run::VerificationEvidence {
9369                    superseded_by: None,
9370                    archived_at: Some(Timestamp::now()),
9371                    fixes_spent: Some(self.state.gate_fixes.len()),
9372                    gate_head: None,
9373                    gate_ran: false,
9374                    gate: Vec::new(),
9375                    e2e: Some(e2e),
9376                });
9377        }
9378        self.state.reviews.clear();
9379        self.state.gate.clear();
9380        self.state.gate_ran = false;
9381        self.state.gate_head = None;
9382        self.state.gate_fixes.clear();
9383        self.state.gate_repair_e2e = None;
9384        self.state.pre_gate.clear();
9385        self.state.pre_gate_commit = None;
9386        self.state.contested_handoff = None;
9387        self.state.land_reruns = None;
9388        self.state.land_approval = None;
9389        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9390        self.state.status = RunStatus::Reviewing;
9391        self.state.save()
9392    }
9393
9394    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9395        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9396        self.state.merge_repairs[index].failure = Some(why.clone());
9397        self.state.status = RunStatus::Blocked;
9398        self.state.event("repair", why);
9399        self.state.save()
9400    }
9401
9402    // -------------------------------------------------------------- helpers
9403
9404    /// Fetch or create a seat, keeping its conversation across nodes.
9405    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9406        if let Some(existing) = self.state.seats.get(key)
9407            && existing.agent == agent
9408        {
9409            return existing.clone();
9410        }
9411        // A seat that changes agent mints its session id from the agent too,
9412        // like a handover: the old agent's uuid is already taken by the CLI.
9413        let fresh = if self.state.seats.contains_key(key) {
9414            handover_seat(key, agent, self.state.next_seat_seed())
9415        } else {
9416            SeatState::new(key, agent, self.state.seed)
9417        };
9418        self.state.seats.insert(key.to_owned(), fresh.clone());
9419        fresh
9420    }
9421
9422    /// The agent now holding seat `key`: `spec`, unless a handover moved the
9423    /// seat to another roster agent, in which case that agent. Nodes that
9424    /// continue a seat's conversation (deliberation, the votes, a reviewer's
9425    /// reconsideration) must keep talking to whoever answered it, not slip
9426    /// back to the agent that failed it.
9427    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9428        match self.state.seats.get(key) {
9429            Some(s) if s.agent != spec.id => {
9430                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9431            }
9432            _ => spec,
9433        }
9434    }
9435
9436    /// A candidate rendered for judging, with the leak policy applied.
9437    fn view(&self, c: &Candidate) -> CandidateView {
9438        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9439            .unwrap_or_default();
9440        let (patch, _) = blind::sanitize_patch(
9441            &format!("candidate {} patch", c.label),
9442            &raw,
9443            &self.state.config.blind,
9444        );
9445        CandidateView {
9446            label: c.label,
9447            branch: c.branch.clone(),
9448            summary: c.summary.clone(),
9449            stat: c.stat.clone(),
9450            patch,
9451        }
9452    }
9453
9454    /// The full candidate set as prompt text, for seats with no live session.
9455    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9456        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9457        prompt::judge(
9458            "(see above)",
9459            &views,
9460            self.roles.judges.len(),
9461            base_short,
9462            "en",
9463        )
9464    }
9465
9466    /// The final-vote prompt with everything a seat that has no session of its
9467    /// own needs: the candidates, the seat's own ranking and reasons, and the
9468    /// anonymised deliberation it took part in (only when there was one, so a
9469    /// handed-over seat never sees more than the seat it replaces did). The
9470    /// `Final vote` heading stays first.
9471    fn vote_prompt_full(
9472        &self,
9473        j: usize,
9474        labels: &[char],
9475        language: &str,
9476        candidates: &[Candidate],
9477        base_short: &str,
9478    ) -> String {
9479        let mut text = format!(
9480            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9481            prompt::final_vote(labels, language),
9482            self.state.instruction,
9483            self.candidate_block(candidates, base_short)
9484        );
9485        if let Some(own) = self
9486            .state
9487            .judgements
9488            .get(j)
9489            .filter(|r| !r.ranking.is_empty())
9490        {
9491            let reasons = own
9492                .reasons
9493                .iter()
9494                .map(|(k, v)| format!("- {k}: {v}"))
9495                .collect::<Vec<_>>()
9496                .join("\n");
9497            text.push_str(&format!(
9498                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9499                own.ranking.iter().collect::<String>(),
9500                if reasons.is_empty() {
9501                    ""
9502                } else {
9503                    ", because:\n"
9504                }
9505            ));
9506        }
9507        if !self.state.deliberation.is_empty() {
9508            text.push_str("\n# What was argued before this vote\n");
9509            for t in self.transcript(&[], j) {
9510                text.push_str(&format!(
9511                    "\n## {}{}\n\n{}\n",
9512                    t.who,
9513                    if t.is_self { " (you)" } else { "" },
9514                    t.body.trim()
9515                ));
9516            }
9517        }
9518        text
9519    }
9520
9521    /// Anonymised transcript for judge `self_idx`.
9522    ///
9523    /// The initial rankings are always the opening statements. Seeding them
9524    /// only when no turn had been taken yet meant every judge after the first
9525    /// argued against a single voice instead of against the actual split — the
9526    /// disagreement is the information, so it is always on the table.
9527    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9528        let mut turns = Vec::new();
9529        for j in &self.state.judgements {
9530            if j.ranking.is_empty() {
9531                continue;
9532            }
9533            let reasons = j
9534                .reasons
9535                .iter()
9536                .map(|(k, v)| format!("- {k}: {v}"))
9537                .collect::<Vec<_>>()
9538                .join("\n");
9539            turns.push(Turn {
9540                who: format!("Judge {} (opening ranking)", j.judge),
9541                is_self: j.judge == self_idx + 1,
9542                body: format!(
9543                    "Ranked {}{}{reasons}",
9544                    j.ranking.iter().collect::<String>(),
9545                    if reasons.is_empty() {
9546                        ""
9547                    } else {
9548                        ", because:\n"
9549                    }
9550                ),
9551            });
9552        }
9553        for t in self
9554            .state
9555            .deliberation
9556            .iter()
9557            .flat_map(|r| r.turns.iter())
9558            .chain(current)
9559        {
9560            turns.push(Turn {
9561                who: format!("Judge {}", t.judge),
9562                is_self: t.judge == self_idx + 1,
9563                body: t.body.clone(),
9564            });
9565        }
9566        turns
9567    }
9568}
9569
9570/// Does this seat still hold the context a follow-up prompt would rely on?
9571fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9572    agent::has_session(spec.kind, seat, sessions)
9573}
9574
9575/// The next entry in `roster` after `start`, never wrapping back to the
9576/// front, whose id is not in `tried` yet.
9577///
9578/// Starts one past `start` rather than at the front of `roster`: `start` is
9579/// the seat's own original position, and a seat whose candidate slot already
9580/// sits on the roster's second entry must fall through to the third next, not
9581/// restart at the first — which is very likely a different candidate's own
9582/// agent already. Never wraps back past `start`, for the same reason: an
9583/// entry earlier in the roster than the seat's own position is almost
9584/// certainly some *other* candidate slot's own agent, and once the tail of
9585/// the roster is exhausted there are no more untried agents for *this* seat
9586/// to fall through to — the caller's fallback chain ends there, exactly as
9587/// "no further untried agents remain in the list for that seat" asks for.
9588///
9589/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
9590/// the same id twice (an operator's `roles.implementers` typo, or a
9591/// `[[agents]]` list reused across roles) must not let
9592/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
9593/// over `roster` either finds an untried id or runs out, so this always
9594/// terminates regardless of duplicates.
9595fn next_untried_in_roster<'a>(
9596    roster: &'a [AgentSpec],
9597    start: usize,
9598    tried: &BTreeSet<String>,
9599) -> Option<&'a AgentSpec> {
9600    roster
9601        .get(start + 1..)?
9602        .iter()
9603        .find(|s| !tried.contains(&s.id))
9604}
9605
9606/// The successor for a seat, shared by all four seat kinds (implement, judge,
9607/// review, advise). `others` holds the ids that *currently* occupy the other
9608/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
9609/// sees them), so roster entries beyond the seat count act as spares: a failed
9610/// seat goes to an agent no other seat holds whenever the roster permits.
9611///
9612/// `carried` is `Some` only for the review loop's carried failure history.
9613/// Order: (1) forward from `start`, never wrapping, untried, not a carried
9614/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
9615/// whole roster: untried and not another seat's occupant; (3) the plain walk
9616/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
9617/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
9618/// a better outcome than an empty seat, and it keeps the answer to "is there
9619/// a successor at all" exactly what it was before occupants were considered,
9620/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
9621/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
9622/// carried failure is a cheaper bet than doubling an agent on the panel, and
9623/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
9624/// the same round are handled one at a time, so a seat later in the batch
9625/// sees an earlier one's new occupant but not yet its own freed agent.
9626fn pick_successor<'a>(
9627    roster: &'a [AgentSpec],
9628    start: usize,
9629    tried: &BTreeSet<String>,
9630    carried: Option<&BTreeSet<String>>,
9631    others: &BTreeSet<String>,
9632) -> Option<&'a AgentSpec> {
9633    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9634    roster
9635        .get(start + 1..)
9636        .and_then(|tail| {
9637            tail.iter()
9638                .filter(free)
9639                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9640        })
9641        .or_else(|| {
9642            carried
9643                .is_some()
9644                .then(|| roster.iter().find(free))
9645                .flatten()
9646        })
9647        .or_else(|| match carried {
9648            Some(c) => next_for_seat(roster, start, tried, c),
9649            None => next_untried_in_roster(roster, start, tried),
9650        })
9651}
9652
9653/// The next agent for a seat that carries its failure history across rounds
9654/// (the review loop). `round_tried` is this round's own bound and starts
9655/// empty every round; `carried_failed` only decides priority.
9656///
9657/// First: an id walking forward from `start`, never wrapping, that is neither
9658/// tried this round nor failed in an earlier one. Only when that is exhausted
9659/// does it rescue: the first roster id (in roster order, so this one *does*
9660/// look before `start`) not yet tried this round, which is by then a carried
9661/// failure. Each id is rescued at most once per round, so it cannot loop.
9662fn next_for_seat<'a>(
9663    roster: &'a [AgentSpec],
9664    start: usize,
9665    round_tried: &BTreeSet<String>,
9666    carried_failed: &BTreeSet<String>,
9667) -> Option<&'a AgentSpec> {
9668    roster
9669        .get(start + 1..)?
9670        .iter()
9671        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9672        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9673}
9674
9675/// Where a reviewer seat starts a round: the agent that last answered it when
9676/// it is still on the roster and not marked failed, else the spec's own agent
9677/// unless it failed, else the next roster agent that has not failed, else the
9678/// spec's own agent again (the whole roster failed). Ids no longer on the
9679/// roster are ignored. An empty roster has no handover, so the spec stands.
9680fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9681    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9682        return spec;
9683    };
9684    let ok = |id: &str| !h.failed.contains(id);
9685    if let Some(last) = h.last_ok.as_deref()
9686        && ok(last)
9687        && let Some(s) = roster.iter().find(|s| s.id == last)
9688    {
9689        return s.clone();
9690    }
9691    if ok(&spec.id) {
9692        return spec;
9693    }
9694    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9695    roster
9696        .get(from + 1..)
9697        .into_iter()
9698        .flatten()
9699        .chain(roster.iter())
9700        .find(|s| ok(&s.id))
9701        .cloned()
9702        .unwrap_or(spec)
9703}
9704
9705/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
9706/// seed so a CLI that mints its session id up front (`--session-id`) never
9707/// reuses the uuid the previous agent already opened under the same seat key.
9708pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9709    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9710}
9711
9712/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
9713/// so the classifier below and the code that builds the message cannot drift.
9714const TIMED_OUT: &str = "timed out";
9715
9716impl FailClass {
9717    /// `None` for an answer; otherwise how the turn failed.
9718    fn of(out: &AgentOutcome) -> Option<Self> {
9719        match out {
9720            AgentOutcome::Ok(_) => None,
9721            AgentOutcome::Quota(_) => Some(Self::Quota),
9722            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9723            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9724            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9725            // Not a failure of the agent. Callers check for it first; `None`
9726            // here means "no class to hand over on".
9727            AgentOutcome::ResourceBlocked(_) => None,
9728        }
9729    }
9730
9731    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
9732    fn stem_word(&self) -> &'static str {
9733        match self {
9734            Self::Quota => "quota",
9735            _ => "handover",
9736        }
9737    }
9738}
9739
9740/// The message's first line with its variable parts removed — digit runs and
9741/// path-like tokens — so "exit code 2" and "exit code 7" read
9742/// as one kind of failure.
9743fn failure_signature(msg: &str) -> String {
9744    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9745    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
9746    // the historical machine key while the process diagnosis uses readable prose.
9747    if let Some(code) = line
9748        .strip_prefix("process exited with no usable output (exit code ")
9749        .and_then(|s| s.strip_suffix(')'))
9750        .and_then(|s| s.parse::<i32>().ok())
9751    {
9752        line = format!("exited with some({code}) and no usable output");
9753    } else if matches!(
9754        line.as_str(),
9755        "process exited with no usable output (no exit code reported)"
9756            | "process exited with no usable output (exit code unavailable)"
9757    ) {
9758        line = "exited with none and no usable output".to_owned();
9759    }
9760    let mut out = Vec::new();
9761    for word in line.split_whitespace() {
9762        if word.contains('/') || word.contains('\\') {
9763            out.push("<path>".to_owned());
9764            continue;
9765        }
9766        let mut w = String::new();
9767        let mut in_digits = false;
9768        for c in word.chars() {
9769            if c.is_ascii_digit() {
9770                if !in_digits {
9771                    w.push('#');
9772                }
9773                in_digits = true;
9774            } else {
9775                in_digits = false;
9776                w.push(c);
9777            }
9778        }
9779        out.push(w);
9780    }
9781    out.join(" ").chars().take(120).collect()
9782}
9783
9784/// Whether a seat that just failed with `cur` may go to the next roster agent.
9785/// A quota or a timeout always may. Any other failure may not when the agent
9786/// before it failed the same way: an error the prompt causes would otherwise
9787/// walk the whole roster. `prev` is the class of the immediately preceding
9788/// agent's failure, so a quota or timeout in between breaks the run of
9789/// identical failures by itself.
9790fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9791    match cur {
9792        FailClass::Quota | FailClass::Timeout => true,
9793        FailClass::Other(_) => prev != Some(cur),
9794    }
9795}
9796
9797/// A short human reason for a failed outcome, for the handover record.
9798fn fail_reason(out: &AgentOutcome) -> String {
9799    match out {
9800        AgentOutcome::Ok(_) => String::new(),
9801        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9802        AgentOutcome::Dropped(o) => format!(
9803            "the CLI dropped the stream ({})",
9804            o.dropped
9805                .as_ref()
9806                .map(|d| d.why.as_str())
9807                .unwrap_or("it ended without delivering its answer")
9808        ),
9809        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9810            e.lines().next().unwrap_or("").chars().take(160).collect()
9811        }
9812    }
9813}
9814
9815/// Note one handover in the run: the structured record and, in the timeline,
9816/// the sentence a person reads. A quota keeps the wording it always had.
9817pub(crate) fn record_handover(
9818    state: &mut RunState,
9819    node: &str,
9820    seat: &str,
9821    from: &str,
9822    to: &str,
9823    class: &FailClass,
9824    reason: &str,
9825) {
9826    let message = if *class == FailClass::Quota {
9827        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9828    } else {
9829        format!("{seat}: handed over {from} -> {to} ({reason})")
9830    };
9831    state.event(node, message);
9832    state.handovers.push(Handover {
9833        at: Timestamp::now(),
9834        node: node.to_owned(),
9835        seat: seat.to_owned(),
9836        from: from.to_owned(),
9837        to: to.to_owned(),
9838        reason: reason.to_owned(),
9839    });
9840}
9841
9842/// Did this reply report running a command whose own CLI never confirmed an
9843/// exit status?
9844///
9845/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9846/// command *finished* (see that type's own doc), so this can only be `true`
9847/// for a command whose completion event carried no readable exit code — not
9848/// for one that simply is not mentioned at all. That is the one signal this
9849/// crate can read, from the same record `state.jobs` renders, about a reply
9850/// standing next to work its own CLI cannot vouch for finishing; it is
9851/// deliberately not a check on the exit code's *value* (a fixer legitimately
9852/// runs a command that fails mid-iteration before it succeeds) and not a
9853/// guess at a command still running in the background (which emits no event
9854/// at all, and so leaves no evidence here to find).
9855fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9856    commands.iter().any(|c| c.exit_code.is_none())
9857}
9858
9859/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9860/// trusted as a verified no-op — the adoption guard's own text-level half.
9861///
9862/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9863/// timed out): a marker only earns the benefit of the doubt from a turn the
9864/// CLI itself vouches for finishing properly, the same house style
9865/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9866/// *fix* report to for `commands`. A candidate that timed out, exited
9867/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9868/// is, whatever prose it wrote — this returns `None` before it ever looks at
9869/// `text`. The remaining guards (the tree really is empty, the evidence is
9870/// non-empty) are the caller's: this only reads what the reply *claimed*.
9871fn verified_noop_claim(
9872    usable: bool,
9873    commands: &[agent::CommandEvidence],
9874    text: &str,
9875) -> Option<String> {
9876    (usable && !has_unconfirmed_command(commands))
9877        .then(|| verdict::verified_noop(text))
9878        .flatten()
9879}
9880
9881fn short(commit: &str) -> String {
9882    commit.chars().take(7).collect()
9883}
9884
9885fn make_executable(path: &Path) -> Result<()> {
9886    #[cfg(unix)]
9887    {
9888        use std::os::unix::fs::PermissionsExt as _;
9889        let mut perms = std::fs::metadata(path)?.permissions();
9890        perms.set_mode(0o755);
9891        std::fs::set_permissions(path, perms)?;
9892    }
9893    #[cfg(not(unix))]
9894    {
9895        let _ = path;
9896    }
9897    Ok(())
9898}
9899
9900/// What every seat in one batch shares: where the answers are attributed, the
9901/// prompt overlay they inherit, and the build cache they are told to use.
9902///
9903/// A struct rather than four more parameters: `wave` also needs the run's
9904/// state (to record who is answering right now) and the attempt number, and
9905/// eight positional arguments is both unreadable and a clippy error.
9906struct WaveCtx<'a> {
9907    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9908    /// paid for it.
9909    run: &'a str,
9910    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9911    node: &'a str,
9912    prompts: &'a Prompts,
9913    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9914    cache: Option<&'a Path>,
9915    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9916    /// `JobRecord::round`. `None` for every other node.
9917    round: Option<usize>,
9918    /// Carry each seat's failed-agent history across waves (the review loop
9919    /// only): start-of-round priority and handover choice read
9920    /// [`RunState::seat_history`], and every answer or failure writes it.
9921    carry_seats: bool,
9922    /// The run's park / interrupt requests, so a wait for a shared resource
9923    /// can be abandoned the moment the operator asks to park.
9924    pauses: Option<(&'a Pause, &'a Pause)>,
9925}
9926
9927/// Shared scheduling policy for operations and spawned seat invocations.
9928fn repository_wait(
9929    state: &RunState,
9930    pauses: Option<(&Pause, &Pause)>,
9931) -> crate::repo_lock::WaitPolicy {
9932    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9933    let inherited = crate::repo_lock::WaitPolicy::current();
9934    crate::repo_lock::WaitPolicy::new(
9935        Duration::from_secs(state.config.graph.timeout_resource_wait),
9936        move || {
9937            inherited.cancelled()
9938                || pauses
9939                    .as_ref()
9940                    .is_some_and(|(a, b)| a.parked() || b.parked())
9941        },
9942    )
9943}
9944
9945fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9946    if error.is::<crate::repo_lock::ResourceBlocked>() {
9947        state.resource_block = Some(crate::run::ResourceBlock {
9948            cause: crate::run::ResourceBlockCause::Other,
9949            node: "repository".into(),
9950            kind: if error.to_string().contains("disk pressure") {
9951                "disk pressure"
9952            } else {
9953                "repository coordination"
9954            }
9955            .into(),
9956            detail: error.to_string(),
9957            at: Timestamp::now(),
9958        });
9959        state.event("resource", error.to_string());
9960    }
9961}
9962
9963async fn coordinate_operation(
9964    state: &mut RunState,
9965    cwd: &Path,
9966    wait: &crate::repo_lock::WaitPolicy,
9967) -> Result<Option<crate::repo_lock::SeatLease>> {
9968    let result =
9969        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9970            .await;
9971    if let Err(e) = &result {
9972        record_repository_block(state, e);
9973    }
9974    result
9975}
9976
9977/// Run one job, honouring the parallelism budget.
9978async fn run_one(
9979    job: SeatJob,
9980    sem: Arc<Semaphore>,
9981    ctx: &WaveCtx<'_>,
9982    state: &mut RunState,
9983    attempt: usize,
9984) -> (SeatState, AgentOutcome) {
9985    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9986        .await
9987        .pop()
9988        .expect("one job in, one result out");
9989    (seat, out)
9990}
9991
9992/// Run every job concurrently, capped by the semaphore, preserving order.
9993///
9994/// Every seat in the batch is recorded into [`RunState::active`] before the
9995/// wave starts and cleared as each answer lands, so the run's own record says
9996/// who is still being waited on rather than only who finished.
9997async fn wave(
9998    jobs: Vec<SeatJob>,
9999    sem: Arc<Semaphore>,
10000    ctx: &WaveCtx<'_>,
10001    state: &mut RunState,
10002    attempt: usize,
10003) -> Vec<(usize, SeatState, AgentOutcome)> {
10004    let WaveCtx {
10005        run,
10006        node,
10007        prompts,
10008        cache,
10009        round,
10010        carry_seats: _,
10011        pauses,
10012    } = *ctx;
10013    for job in &jobs {
10014        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10015    }
10016    if let Err(e) = state.save() {
10017        // A failed persist of "who is answering right now" must not abort the
10018        // wave: the seats are already being asked, and the alternative is
10019        // losing the answers to save a status line nobody may even be
10020        // watching.
10021        tracing::warn!("could not persist in-progress seats: {e:#}");
10022    }
10023    // Hold the shared build cache's lease for the whole batch, not per job:
10024    // several candidates (an implement wave) or a fixer legitimately share
10025    // one cache concurrently within this run, and that stays untouched — a
10026    // single lease taken once for the whole wave and released once it is
10027    // done is what stops a *different* borrower (another run's own wave, its
10028    // e2e/gate, a human's `magi review`) from interleaving a build into the
10029    // same directory while this one is in flight. Best-effort, not
10030    // all-or-nothing: a wave that cannot get the lease within its own
10031    // longest job's budget still runs — an hour of paid implementer calls is
10032    // not thrown away over cache contention — but every write-allowed seat
10033    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10034    // below), the same fallback a read-only seat always gets, rather than
10035    // building into a directory this run was never granted. The identity
10036    // record is still invalidated below either way, so the next tracked
10037    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10038    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10039    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10040    // never borrowed from a job's agent budget (and never charged to it either,
10041    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10042    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10043    let cache_guard = if let Some(cache_dir) = cache {
10044        if jobs_had_a_writer {
10045            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10046            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10047            match acquire_cache_lease(
10048                state,
10049                cache_dir,
10050                &owner,
10051                resource_limit,
10052                node,
10053                node,
10054                &cancel,
10055            )
10056            .await
10057            {
10058                Ok(Some(g)) => Some(g),
10059                // Parking: leave without spawning anything. The run parks at
10060                // the next boundary and the seats are asked afresh on resume.
10061                Ok(None) => {
10062                    if let Some(block) = &mut state.resource_block {
10063                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10064                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10065                        };
10066                    }
10067                    return block_wave(
10068                        jobs,
10069                        state,
10070                        node,
10071                        attempt,
10072                        "parked while waiting for the shared build cache",
10073                    );
10074                }
10075                // An unreadable lease is not contention: keep the old
10076                // fallback (no shared cache for this wave's seats).
10077                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10078                Err(e) => {
10079                    // Writing seats may not build into a cache this run does
10080                    // not hold, and an agent that cannot be given one is not
10081                    // asked at all: the run stops resumably instead.
10082                    let why = format!("{e:#}");
10083                    state.event(
10084                        "resource",
10085                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10086                    );
10087                    return block_wave(jobs, state, node, attempt, &why);
10088                }
10089            }
10090        } else {
10091            None
10092        }
10093    } else {
10094        None
10095    };
10096    // Every job keeps its whole budget: the wait above was not the agent's.
10097    // Restart the seats' clocks so "started" means "being spawned now".
10098    state.seats_spawning(node, attempt);
10099    if let Err(e) = state.save() {
10100        tracing::warn!("could not persist the start of a wave: {e:#}");
10101    }
10102    // Queueing behind this wave's own seats for a parallelism slot is normal
10103    // (each of them is bounded by its own agent timeout); only a wait beyond
10104    // all of them plus the resource limit means the slots never came back.
10105    let slot_limit = resource_limit
10106        + jobs
10107            .iter()
10108            .map(|j| j.timeout)
10109            .max()
10110            .unwrap_or_default()
10111            .saturating_mul(jobs.len() as u32);
10112    let mut set = tokio::task::JoinSet::new();
10113    // Seats announce themselves here the moment they hold a parallelism
10114    // permit, i.e. just before their CLI is spawned.
10115    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10116    let overlay = prompts.overlay(node);
10117    let repository_wait = repository_wait(state, pauses);
10118    for (i, mut job) in jobs.into_iter().enumerate() {
10119        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10120        if cache.is_some() {
10121            job.prompt.push('\n');
10122            job.prompt
10123                .push_str(&prompt::build_cache_note(node, job.allow_write));
10124        }
10125        let sem = Arc::clone(&sem);
10126        let run = run.to_owned();
10127        let node = node.to_owned();
10128        // Only implementers were told about the task's attachments, so only
10129        // their seats get the directory widened for reading.
10130        let attachments = if node == "implement" {
10131            state.attachments.clone()
10132        } else {
10133            Vec::new()
10134        };
10135        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10136        // `prompt::build_cache_note`'s doc for why setting it anyway is
10137        // exactly how a sandboxed reviewer's write refusal got reported as a
10138        // defect in the patch, not a property of its own seat. And a
10139        // write-allowed one is handed it only when the lease above was
10140        // actually acquired: a wave that could not get it (`cache_guard` is
10141        // `None`, see its own comment) must not send seats to build into a
10142        // directory this run does not hold - that is the exact concurrent,
10143        // unmanaged-write race this module exists to prevent, not something
10144        // "proceeding anyway" is allowed to reintroduce.
10145        let cache = cache
10146            .filter(|_| job.allow_write && cache_guard.is_some())
10147            .map(Path::to_path_buf);
10148        let spawned_tx = spawned_tx.clone();
10149        let repository_wait = repository_wait.clone();
10150        let coordination = state.config.coordination.clone();
10151        let fetched_base = state.fetched_base.clone();
10152        set.spawn(async move {
10153            // A wait for a parallelism slot is scheduling, not the agent: it
10154            // is bounded by its own limit and ends as a resource block, never
10155            // as a timeout of the agent that was not started.
10156            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10157                Ok(p) => p,
10158                Err(_) => {
10159                    return (
10160                        i,
10161                        job.seat,
10162                        AgentOutcome::ResourceBlocked(format!(
10163                            "no parallel slot became free within {}s",
10164                            slot_limit.as_secs()
10165                        )),
10166                        false,
10167                    );
10168                }
10169            };
10170            spawned_tx.send(job.seat.key.clone()).ok();
10171            let mut seat = job.seat;
10172            let out = repository_wait
10173                .scope(agent::invoke(
10174                    &job.spec,
10175                    &mut seat,
10176                    &Invocation {
10177                        coordination: coordination.as_ref(),
10178                        fetched_base: fetched_base.as_ref(),
10179                        cwd: &job.cwd,
10180                        prompt: &job.prompt,
10181                        timeout: job.timeout,
10182                        allow_write: job.allow_write,
10183                        sessions: job.sessions,
10184                        artifacts: &job.artifacts,
10185                        stem: &job.stem,
10186                        run: &run,
10187                        node: &node,
10188                        cache_dir: cache.as_deref(),
10189                        attachments: &attachments,
10190                        writable: &[],
10191                    },
10192                ))
10193                .await;
10194            let interrupted = out
10195                .as_ref()
10196                .err()
10197                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10198            let out = match out {
10199                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10200                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10201                // Billed work the CLI failed to hand over is not an ordinary
10202                // failure, but its text is the CLI's raw error JSON, not an
10203                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10204                // read it as one by forgetting to check. `usable()` is always
10205                // false here (dropped implies an empty response), so this has
10206                // to be checked before the catch-all `Failed` below or the
10207                // one shape this exists for is lost with the rest.
10208                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10209                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10210                Ok(o) => AgentOutcome::Failed(format!(
10211                    "process exited with no usable output ({})",
10212                    crate::notices::exit_status(o.exit_code)
10213                )),
10214                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10215                    AgentOutcome::ResourceBlocked(e.to_string())
10216                }
10217                Err(e) => AgentOutcome::Failed(e.to_string()),
10218            };
10219            (i, seat, out, interrupted)
10220        });
10221    }
10222    drop(spawned_tx);
10223    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10224    loop {
10225        let joined = tokio::select! {
10226            Some(key) = spawned_rx.recv() => {
10227                // The clock of this seat starts now, not when it queued.
10228                state.seat_spawned(&key);
10229                if let Err(e) = state.save() {
10230                    tracing::warn!("could not persist a seat's start: {e:#}");
10231                }
10232                continue;
10233            }
10234            joined = set.join_next() => joined,
10235        };
10236        let Some(joined) = joined else {
10237            break;
10238        };
10239        let (i, seat, out, interrupted) = match joined {
10240            Ok(v) => v,
10241            // No seat to clear: a panicked task never reported which one it
10242            // was. The defensive sweep below this loop is what stops that
10243            // seat's `active` entry from surviving forever.
10244            Err(e) => {
10245                tracing::error!("agent task panicked: {e}");
10246                continue;
10247            }
10248        };
10249        state.seat_finished(&seat.key);
10250        if interrupted
10251            && node == "implement"
10252            && let Some(label) = state
10253                .candidates
10254                .iter()
10255                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10256                .map(|candidate| candidate.label)
10257        {
10258            state.candidate_preservation.insert(
10259                label,
10260                "interrupted implementation awaiting persistence".into(),
10261            );
10262            state.interrupted_implementations.insert(label);
10263        }
10264        if let AgentOutcome::ResourceBlocked(why) = &out
10265            && state.resource_block.is_none()
10266        {
10267            state.resource_block = Some(crate::run::ResourceBlock {
10268                cause: crate::run::ResourceBlockCause::Other,
10269                node: node.to_owned(),
10270                kind: if why.contains("disk pressure") {
10271                    "disk pressure"
10272                } else if why.contains("repository coordination") {
10273                    "repository coordination"
10274                } else {
10275                    "parallel slot"
10276                }
10277                .to_owned(),
10278                detail: why.clone(),
10279                at: Timestamp::now(),
10280            });
10281            let activity = if interrupted {
10282                "agent execution interrupted"
10283            } else {
10284                "no agent was started"
10285            };
10286            state.event("resource", format!("{node}: {activity} - {why}"));
10287        }
10288        record_jobs(state, node, round, &seat.key, &out);
10289        if let Err(e) = state.save() {
10290            tracing::warn!("could not persist a seat's completion: {e:#}");
10291        }
10292        if collected.len() <= i {
10293            collected.resize_with(i + 1, || None);
10294        }
10295        collected[i] = Some((i, seat, out));
10296    }
10297    // Belt-and-braces for the panic branch above: every seat this exact batch
10298    // started shares this `(node, attempt)` pair, and every seat that finished
10299    // normally already cleared itself, so anything left tagged with it here
10300    // can only be a panicked task's leftover. Cleared unconditionally rather
10301    // than left to read as still answering forever.
10302    if state
10303        .active
10304        .values()
10305        .any(|a| a.node == node && a.attempt == attempt)
10306    {
10307        state
10308            .active
10309            .retain(|_, a| !(a.node == node && a.attempt == attempt));
10310        if let Err(e) = state.save() {
10311            tracing::warn!("could not persist the end of a wave: {e:#}");
10312        }
10313    }
10314    // Whether or not the lease above was actually held, several worktrees
10315    // may just have built into the cache with nothing here able to name one
10316    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
10317    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
10318    // clean it might not have strictly needed; trusting a stale match would
10319    // cost it a wrong answer.
10320    if let Some(cache_dir) = cache
10321        && jobs_had_a_writer
10322    {
10323        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10324    }
10325    if let Some(guard) = cache_guard {
10326        guard.release();
10327    }
10328    collected.into_iter().flatten().collect()
10329}
10330
10331/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
10332/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
10333/// this is data collection, not the fix-specific completion contract in
10334/// [`Runner::continue_fix_report`], and applies regardless of which node
10335/// asked.
10336///
10337/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
10338/// evidence from; `Failed` does not, and correctly contributes nothing — a
10339/// timeout or crash is not itself evidence about a command the seat may have
10340/// started.
10341fn record_jobs(
10342    state: &mut RunState,
10343    node: &str,
10344    round: Option<usize>,
10345    seat: &str,
10346    out: &AgentOutcome,
10347) {
10348    let commands: &[agent::CommandEvidence] = match out {
10349        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10350        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10351    };
10352    let checked_at = Timestamp::now();
10353    for c in commands {
10354        state.jobs.push(JobRecord {
10355            node: node.to_owned(),
10356            round,
10357            seat: seat.to_owned(),
10358            id: c.id.clone(),
10359            description: c.description.clone(),
10360            checked_at,
10361            status: match c.exit_code {
10362                Some(0) => JobStatus::Completed,
10363                Some(_) => JobStatus::Failed,
10364                None => JobStatus::Unknown,
10365            },
10366            exit_code: c.exit_code,
10367            result_summary: c.result_summary.clone(),
10368            source: c.source.clone(),
10369        });
10370    }
10371}
10372
10373/// Is a review round clean, given how many reviewer seats answered against
10374/// how many the round expected?
10375///
10376/// A seat that never answered (timeout, crash, unparsable output) is not a
10377/// seat that read the patch and found nothing — treating it as such is
10378/// exactly the bug this function exists to close. Under the default `block`
10379/// policy a missing seat can never be clean; `warn` still requires the seats
10380/// that *did* answer to have found nothing blocking and verification to be
10381/// green.
10382///
10383/// `quota_missing` narrows that `block` default for exactly one cause of
10384/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
10385/// a session limit lifts by the very next round buys nothing — the seat is
10386/// asked again with the same quota — so once every missing seat is accounted
10387/// for by a quota loss (and at least one seat *did* answer, so a decision has
10388/// something to rest on) the round is decided on the panel that could answer,
10389/// same as `warn` would. A panel that lost every seat to quota is not
10390/// decided here: `answered == 0` falls through to the existing `block`
10391/// fallback so a fully collapsed panel still waits rather than landing on no
10392/// review at all.
10393fn round_is_clean(
10394    blocking: usize,
10395    e2e_ok: bool,
10396    answered: usize,
10397    expected: usize,
10398    quota_missing: usize,
10399    policy: IncompleteReviewPolicy,
10400) -> bool {
10401    if blocking != 0 || !e2e_ok {
10402        return false;
10403    }
10404    if answered == expected || policy == IncompleteReviewPolicy::Warn {
10405        return true;
10406    }
10407    answered > 0 && expected - answered <= quota_missing
10408}
10409
10410/// The review loop's own conclusion, derived entirely from its persisted
10411/// round records and the round budget that produced them — never from
10412/// `status`, so a reentry (or `gate`/`merge` reading it independently)
10413/// recomputes the identical answer regardless of what an earlier node in the
10414/// same walk, or a previous walk, did to `status`.
10415///
10416/// `None` while more rounds remain to try, including when review never ran
10417/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
10418/// gone clean, or the budget is spent, or the tree has stopped moving (see
10419/// [`STAGNANT_LIMIT`]), the answer is one of two things:
10420///
10421/// - An incomplete panel that raised nothing is missing input, not a
10422///   verified tree — never a hand-off candidate, whatever verification said
10423///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
10424/// - Otherwise, green e2e on the last round hands off (see
10425///   [`Runner::stop_reviewing`]); red e2e blocks.
10426///
10427/// A last round whose own verification is still `ResourceBlocked` — magi
10428/// itself never got a command to run, not evidence the patch is broken —
10429/// is neither: this returns `None` for it too, the same as "more rounds
10430/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
10431/// handling of that shape) instead of this cheap recomputation guessing a
10432/// verdict a real attempt never produced.
10433pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10434    if max_rounds == 0
10435        || reviews.last().is_some_and(|r| {
10436            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10437        })
10438    {
10439        return Some(RunStatus::Gating);
10440    }
10441    let last = reviews.last()?;
10442    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10443    if reviews.len() < max_rounds && !stagnant {
10444        return None;
10445    }
10446    if last.incomplete() && last.blocking == 0 {
10447        return Some(RunStatus::Blocked);
10448    }
10449    if last.e2e_status() == E2eStatus::ResourceBlocked
10450        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10451    {
10452        return None;
10453    }
10454    Some(if last.e2e.iter().all(CommandOutcome::ok) {
10455        RunStatus::Gating
10456    } else {
10457        RunStatus::Blocked
10458    })
10459}
10460
10461/// How long a re-ask may take, given the budget the first attempt had.
10462///
10463/// A `nudged` retry is a request to restate an answer the seat has already
10464/// worked out: it carries no new work, so it does not deserve the original
10465/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
10466/// 133 seconds while a third sat for over ten minutes on a resumed session
10467/// holding 410 KB of prior output - and because the retry had inherited the
10468/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
10469/// judging round whose other seats were long finished.
10470///
10471/// A quarter of the budget, with a floor so that a deliberately short timeout
10472/// does not collapse to nothing. A retry that re-sends the whole prompt
10473/// (because the seat kept no context) is the original job again, and keeps the
10474/// original budget.
10475pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10476    if nudged {
10477        (full / 4).max(Duration::from_secs(120)).min(full)
10478    } else {
10479        full
10480    }
10481}
10482
10483/// Run a wave and parse each reply, re-asking the seats whose reply was
10484/// unusable.
10485///
10486/// The re-ask is a nudge rather than the whole prompt again when the seat still
10487/// holds its conversation, which is the difference between a cheap retry and
10488/// paying for the entire candidate set twice.
10489///
10490/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
10491/// successor in `roster` is handed to it instead of being re-asked: the
10492/// handover is the retry. A seat with no successor left (a single-agent
10493/// roster, the roster's tail) is nudged as before, up to `retries` times. So
10494/// the asks to one seat in one node number at most
10495/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
10496/// is asked once (a dropped stream is nudged first), and only the last agent
10497/// of the chain gets the `retries` same-agent nudges. Each roster agent is
10498/// tried at most once per seat, walking forward from the seat's own position and never wrapping
10499/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
10500/// other failure stops the chain when the previous agent failed the same way
10501/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
10502/// [`has_context`] is false and the job's own full prompt and full budget are
10503/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
10504/// intermediate ones are not losses) and is returned as a failure like any
10505/// other absent seat — the caller decides whether the panel still has a
10506/// quorum. An empty `roster` disables handover: failures are nudged as they
10507/// always were, and a quota is simply lost. A reply that fails to parse or
10508/// validate is the prompt's doing and is only ever nudged, never handed over.
10509///
10510/// The returned [`SeatState`] names the agent that answered (or tried last).
10511#[allow(clippy::too_many_arguments)]
10512async fn ask_json_wave<T>(
10513    jobs: Vec<SeatJob>,
10514    sem: Arc<Semaphore>,
10515    retries: usize,
10516    roster: &[AgentSpec],
10517    ctx: &WaveCtx<'_>,
10518    losses: &mut Vec<QuotaLoss>,
10519    state: &mut RunState,
10520    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10521) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10522where
10523    T: serde::de::DeserializeOwned + Send + 'static,
10524{
10525    ask_wave_with(
10526        jobs,
10527        sem,
10528        retries,
10529        roster,
10530        ctx,
10531        losses,
10532        state,
10533        &|text: &str| {
10534            let v = verdict::extract_json::<T>(text)?;
10535            validate(&v)?;
10536            Ok(v)
10537        },
10538    )
10539    .await
10540}
10541
10542/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
10543/// a node whose answer is prose (deliberation) shares the same handover,
10544/// failure classification, quota bookkeeping and bounds instead of a copy.
10545///
10546/// A seat handed to another roster agent is sent the job's `handover` prompt
10547/// (when it has one) rather than `prompt`: the new agent has no session, so a
10548/// resume-style prompt would be incomplete. That holds for the handover ask
10549/// and for every nudge to that agent whose `has_context` is false.
10550#[allow(clippy::too_many_arguments)]
10551async fn ask_wave_with<T>(
10552    jobs: Vec<SeatJob>,
10553    sem: Arc<Semaphore>,
10554    retries: usize,
10555    roster: &[AgentSpec],
10556    ctx: &WaveCtx<'_>,
10557    losses: &mut Vec<QuotaLoss>,
10558    state: &mut RunState,
10559    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10560) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10561where
10562    T: Send + 'static,
10563{
10564    let n = jobs.len();
10565    let originals: Vec<SeatJob> = jobs;
10566    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10567    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10568    // Nudges each seat's *current* agent has taken — 0 for a first-ask
10569    // answer, N once it has gone through N nudges. Read back once this
10570    // returns, so a caller building a history record (`ReviewRecord`) can
10571    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
10572    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
10573    // field's own doc.
10574    let mut nudges: Vec<usize> = vec![0; n];
10575    // The agent now occupying each seat, the ids it has already been through,
10576    // where in the roster the walk began, the class of the last failure, and
10577    // the stem word of a handover not yet asked (full prompt, full budget).
10578    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10579    let mut tried: Vec<BTreeSet<String>> = specs
10580        .iter()
10581        .map(|s| BTreeSet::from([s.id.clone()]))
10582        .collect();
10583    let starts: Vec<usize> = specs
10584        .iter()
10585        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10586        .collect();
10587    let carry = ctx.carry_seats && !roster.is_empty();
10588    // Carried across rounds: ids that failed the seat earlier, and how the
10589    // last failure went (so a repeat of it is not handed over again).
10590    let carried: Vec<BTreeSet<String>> = originals
10591        .iter()
10592        .map(|j| {
10593            state
10594                .seat_history
10595                .get(&j.seat.key)
10596                .filter(|_| carry)
10597                .map(|h| h.failed.clone())
10598                .unwrap_or_default()
10599        })
10600        .collect();
10601    let mut prev: Vec<Option<FailClass>> = originals
10602        .iter()
10603        .map(|j| {
10604            state
10605                .seat_history
10606                .get(&j.seat.key)
10607                .filter(|_| carry)
10608                .and_then(|h| h.last_fail.clone())
10609        })
10610        .collect();
10611    let next_agent =
10612        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10613            let others: BTreeSet<String> = specs
10614                .iter()
10615                .enumerate()
10616                .filter(|(j, _)| *j != i)
10617                .map(|(_, s)| s.id.clone())
10618                .collect();
10619            pick_successor(
10620                roster,
10621                starts[i],
10622                tried,
10623                carry.then(|| &carried[i]),
10624                &others,
10625            )
10626            .cloned()
10627        };
10628    let mut fresh: Vec<Option<String>> = vec![None; n];
10629    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
10630    let mut pending: Vec<usize> = (0..n).collect();
10631
10632    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
10633    // asks: an agent with a successor is asked once and handed over, and only
10634    // a seat with no successor spends `retries` nudges on the same agent. This
10635    // only guarantees the loop's own termination whatever those say.
10636    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10637    for round in 0..max_rounds {
10638        if pending.is_empty() {
10639            break;
10640        }
10641        let mut batch = Vec::with_capacity(pending.len());
10642        let mut renudged: Vec<&str> = Vec::new();
10643        for &i in &pending {
10644            let src = &originals[i];
10645            // A seat now held by another agent than the job named has no
10646            // session of its own: it gets the full-context prompt whenever
10647            // it is asked in full (the handover ask, a nudge it cannot
10648            // resume).
10649            let full: &str = match &src.handover {
10650                Some(h) if specs[i].id != src.spec.id => h,
10651                _ => &src.prompt,
10652            };
10653            // The prompt and the budget are one decision: a nudge restates
10654            // finished work, a re-sent prompt redoes it.
10655            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10656                (
10657                    full.to_owned(),
10658                    src.timeout,
10659                    format!("{}-{word}-{}", src.stem, specs[i].id),
10660                )
10661            } else if nudges[i] == 0 {
10662                (full.to_owned(), src.timeout, src.stem.clone())
10663            } else {
10664                renudged.push(src.seat.key.as_str());
10665                let why = done[i]
10666                    .as_ref()
10667                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
10668                    .unwrap_or_else(|| "no parsable answer".to_owned());
10669                let nudge = prompt::nudge(&why);
10670                let nudged = has_context(&specs[i], &seats[i], src.sessions);
10671                let prompt = if nudged {
10672                    nudge
10673                } else {
10674                    format!("{full}\n\n---\n\n{nudge}")
10675                };
10676                (
10677                    prompt,
10678                    retry_budget(src.timeout, nudged),
10679                    format!("{}-retry{}", src.stem, nudges[i]),
10680                )
10681            };
10682            batch.push(SeatJob {
10683                spec: specs[i].clone(),
10684                seat: seats[i].clone(),
10685                cwd: src.cwd.clone(),
10686                prompt,
10687                timeout,
10688                allow_write: src.allow_write,
10689                sessions: src.sessions,
10690                artifacts: src.artifacts.clone(),
10691                stem,
10692                handover: None,
10693            });
10694        }
10695
10696        if !renudged.is_empty() {
10697            state.event(
10698                ctx.node,
10699                format!("retry {round}: re-asking {}", renudged.join(", ")),
10700            );
10701        }
10702        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10703        let mut still = Vec::new();
10704        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10705            seats[i] = seat;
10706            // No agent was asked: nothing to hand over, nudge, remember or
10707            // charge. The caller sees `resource_block` and stops the node.
10708            if let AgentOutcome::ResourceBlocked(why) = &out {
10709                done[i] = Some(Err(anyhow::anyhow!("{why}")));
10710                continue;
10711            }
10712            let class = FailClass::of(&out);
10713            // A dropped stream is nudged first (the conversation is still
10714            // there to pick up); only a seat whose nudges are spent hands over.
10715            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10716                && nudges[i] < retries
10717                && !roster.is_empty();
10718            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10719                let next = should_hand_over(prev[i].as_ref(), &cur)
10720                    .then(|| next_agent(i, &tried[i], &specs))
10721                    .flatten();
10722                if carry {
10723                    let h = state
10724                        .seat_history
10725                        .entry(originals[i].seat.key.clone())
10726                        .or_default();
10727                    h.failed.insert(specs[i].id.clone());
10728                    h.last_fail = Some(cur.clone());
10729                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10730                        h.last_ok = None;
10731                    }
10732                    // Saved before the next agent is asked, so a restart in
10733                    // between does not forget who failed.
10734                    if let Err(e) = state.save() {
10735                        tracing::warn!("could not persist a seat's failure history: {e:#}");
10736                    }
10737                }
10738                if let Some(next) = next {
10739                    record_handover(
10740                        state,
10741                        ctx.node,
10742                        &originals[i].seat.key,
10743                        &specs[i].id,
10744                        &next.id,
10745                        &cur,
10746                        &fail_reason(&out),
10747                    );
10748                    tried[i].insert(next.id.clone());
10749                    prev[i] = Some(cur.clone());
10750                    seats[i] =
10751                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10752                    specs[i] = next;
10753                    fresh[i] = Some(cur.stem_word().to_owned());
10754                    nudges[i] = 0;
10755                    done[i] = Some(Err(anyhow::anyhow!(
10756                        "handed over after: {}",
10757                        fail_reason(&out)
10758                    )));
10759                    still.push(i);
10760                    continue;
10761                }
10762            }
10763            if carry
10764                && !nudge_first
10765                && let Some(cur) = class.clone()
10766            {
10767                // The chain ended here (no successor, or a repeated failure).
10768                let h = state
10769                    .seat_history
10770                    .entry(originals[i].seat.key.clone())
10771                    .or_default();
10772                h.failed.insert(specs[i].id.clone());
10773                h.last_fail = Some(cur);
10774            }
10775            let parsed = match out {
10776                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10777                AgentOutcome::Quota(o) => {
10778                    last_quota[i] = o.quota.clone();
10779                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10780                }
10781                // Not a parseable answer: the nudge loop re-asks it, which is
10782                // exactly what a dropped stream needs. Just don't hand its raw
10783                // error JSON to `extract_json`.
10784                AgentOutcome::Dropped(o) => {
10785                    let why = o
10786                        .dropped
10787                        .as_ref()
10788                        .map(|d| d.why.as_str())
10789                        .unwrap_or("the CLI ended the stream without delivering its answer");
10790                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10791                }
10792                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10793                    Err(anyhow::anyhow!(e))
10794                }
10795            };
10796            let quota = class == Some(FailClass::Quota);
10797            let failed = parsed.is_err();
10798            if !failed && matches!(ctx.node, "judge" | "vote") {
10799                // A validated answer restores this seat's quorum presence,
10800                // including on the normal path after an all-seat block.
10801                state
10802                    .quota
10803                    .retain(|loss| loss.seat != originals[i].seat.key);
10804            }
10805            done[i] = Some(parsed);
10806            if carry && !failed {
10807                let h = state
10808                    .seat_history
10809                    .entry(originals[i].seat.key.clone())
10810                    .or_default();
10811                h.failed.remove(&specs[i].id);
10812                h.last_ok = Some(specs[i].id.clone());
10813                h.last_fail = None;
10814            }
10815            // Do not re-ask a rate-limited seat (quota) — a retry is known to
10816            // fail the same way; and never re-ask a seat that already parsed.
10817            // A failed agent that still has a successor is not re-asked
10818            // either: the handover was its remedy and has just been refused
10819            // (the chain stops on a repeated failure class). A seat with no
10820            // successor left (a single-agent roster, the roster's tail, or an
10821            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10822            let agent_failure = class.is_some()
10823                && !nudge_first
10824                && !roster.is_empty()
10825                && next_agent(i, &tried[i], &specs).is_some();
10826            if failed && !quota && !agent_failure && nudges[i] < retries {
10827                nudges[i] += 1;
10828                still.push(i);
10829            }
10830        }
10831        pending = still;
10832    }
10833
10834    // One loss per seat whose chain ended on a quota: the intermediate ones
10835    // were absorbed by a handover and are not losses.
10836    for (i, q) in last_quota.into_iter().enumerate() {
10837        if let Some(quota) = q {
10838            losses.push(QuotaLoss::from_report(
10839                originals[i].seat.key.clone(),
10840                ctx.node,
10841                Some(&quota),
10842            ));
10843        }
10844    }
10845
10846    // A stalled quorum already has its own recovery checkpoint. Refresh
10847    // its missing seats before deciding whether it can resume.
10848    if state.status != RunStatus::Stalled
10849        && ctx.node != "deliberate"
10850        && n > 0
10851        && losses.len() == n
10852        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
10853    {
10854        block_machine_outage(state, ctx.node, losses);
10855        state.quota.append(losses);
10856    }
10857
10858    seats
10859        .into_iter()
10860        .zip(done)
10861        .zip(nudges)
10862        .map(|((seat, res), attempts)| {
10863            (
10864                seat,
10865                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10866                attempts,
10867            )
10868        })
10869        .collect()
10870}
10871
10872/// A node with no available agent has no task verdict. Preserve the node
10873/// checkpoint and record why a subsequent explicit resume is needed.
10874pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
10875    let mut auth: Vec<_> = losses
10876        .iter()
10877        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
10878        .map(|loss| loss.agent.clone())
10879        .collect();
10880    auth.sort();
10881    auth.dedup();
10882    let (kind, detail) = if auth.is_empty() {
10883        (
10884            "agent usage limit",
10885            "All seats reached their agent usage limits; resume after usage is available"
10886                .to_owned(),
10887        )
10888    } else {
10889        (
10890            "agent authentication",
10891            format!(
10892                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
10893                auth.join(", ")
10894            ),
10895        )
10896    };
10897    state.resource_block = Some(crate::run::ResourceBlock {
10898        node: node.to_owned(),
10899        kind: kind.to_owned(),
10900        cause: crate::run::ResourceBlockCause::Other,
10901        detail: detail.clone(),
10902        at: Timestamp::now(),
10903    });
10904    state.event(node, detail);
10905}
10906
10907/// Did the wave end without asking any agent (see [`block_wave`])?
10908fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10909    results
10910        .iter()
10911        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10912}
10913
10914/// Settle a wave that never spawned anything: every job answers
10915/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10916/// else about the seat (session, history, quota) is touched.
10917fn block_wave(
10918    jobs: Vec<SeatJob>,
10919    state: &mut RunState,
10920    node: &str,
10921    attempt: usize,
10922    why: &str,
10923) -> Vec<(usize, SeatState, AgentOutcome)> {
10924    state.resource_wait = None;
10925    state
10926        .active
10927        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10928    if let Err(e) = state.save() {
10929        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10930    }
10931    jobs.into_iter()
10932        .enumerate()
10933        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10934        .collect()
10935}
10936
10937/// Acquire the shared build cache's lease, waiting out contention within
10938/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10939/// unbounded wait is never acceptable).
10940///
10941/// A first, non-blocking check happens before ever waiting; if it finds the
10942/// lease busy, that fact is logged as a `verify` event *and* flushed with
10943/// [`RunState::save`] immediately — not only once the wait finally succeeds
10944/// or gives up — so a `magi show` run by a different process while this one
10945/// is still waiting reads a `run.json` that says so, rather than whatever it
10946/// looked like before the wait started. The same applies to the terminal
10947/// failure: logged and saved before this returns `Err`, so a caller that
10948/// could not get the lease at all still leaves a legible record of why.
10949async fn acquire_cache_lease(
10950    state: &mut RunState,
10951    cache_dir: &Path,
10952    owner: &crate::cache::Owner,
10953    budget: Duration,
10954    node: &str,
10955    context: &str,
10956    cancel: &(dyn Fn() -> bool + Sync),
10957) -> Result<Option<crate::cache::Guard>> {
10958    let home = crate::run::home();
10959    let started = Instant::now();
10960    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10961        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10962        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10963        Err(e) => {
10964            state.event(
10965                "verify",
10966                format!("{context}: could not check the shared build cache: {e:#}"),
10967            );
10968            if let Err(e2) = state.save() {
10969                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10970            }
10971            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10972        }
10973    };
10974    state.event(
10975        "verify",
10976        format!(
10977            "{context}: waiting for the shared build cache at {} ({})",
10978            cache_dir.display(),
10979            busy.describe()
10980        ),
10981    );
10982    // Seats of this node are waiting, not running: say so, so the countdown a
10983    // reader sees only ever measures an agent that has been spawned.
10984    state.resource_wait = Some(crate::run::ResourceWait {
10985        node: context.to_owned(),
10986        kind: "shared build cache".to_owned(),
10987        holder: busy.describe(),
10988        since: Timestamp::now(),
10989        limit_secs: budget.as_secs(),
10990    });
10991    if let Err(e) = state.save() {
10992        tracing::warn!("could not persist a cache wait: {e:#}");
10993    }
10994    let remaining = budget.saturating_sub(started.elapsed());
10995    let waited = crate::cache::wait_for_unless(
10996        &home,
10997        cache_dir,
10998        owner,
10999        remaining,
11000        Duration::from_secs(5),
11001        cancel,
11002    )
11003    .await;
11004    state.resource_wait = None;
11005    let cause = match &waited {
11006        Ok(Some(_)) => None,
11007        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11008        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11009            Some(crate::run::ResourceBlockCause::Timeout {
11010                waited_secs: started.elapsed().as_secs(),
11011                limit_secs: budget.as_secs(),
11012            })
11013        }
11014        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11015    };
11016    if let Some(cause) = cause {
11017        let detail = match &waited {
11018            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11019            Err(error) => format!("{error:#}"),
11020        };
11021        state.resource_block = Some(crate::run::ResourceBlock {
11022            cause,
11023            node: node.to_owned(),
11024            kind: "shared build cache".into(),
11025            detail,
11026            at: Timestamp::now(),
11027        });
11028        state.event(
11029            "resource",
11030            state.resource_block.as_ref().unwrap().description(),
11031        );
11032    }
11033    if let Err(error) = state.save() {
11034        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11035    }
11036    waited
11037}
11038
11039/// The lease state could not be read at all (unreadable lease file, a
11040/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11041/// known to be contended, so a wave degrades to running without a shared cache
11042/// instead of stopping on a hold that [`crate::triage`] could never release.
11043#[derive(Debug)]
11044struct CacheCheckError(String);
11045
11046impl std::fmt::Display for CacheCheckError {
11047    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11048        f.write_str(&self.0)
11049    }
11050}
11051
11052impl std::error::Error for CacheCheckError {}
11053
11054/// Keep a specific acquisition outcome when the caller adds command evidence.
11055fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11056    state
11057        .resource_block
11058        .get_or_insert_with(|| crate::run::ResourceBlock {
11059            cause: crate::run::ResourceBlockCause::Other,
11060            node: node.to_owned(),
11061            kind: "shared build cache".into(),
11062            detail: format!("{error:#}"),
11063            at: Timestamp::now(),
11064        });
11065    if let Err(error) = state.save() {
11066        tracing::warn!("could not persist cache resource block: {error:#}");
11067    }
11068}
11069
11070/// Run `body` — a verify command batch — while holding the shared build
11071/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11072/// never interleave with another borrower's build against the same
11073/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11074/// timeout, or a human's own `magi review`. See the `cache` module doc for
11075/// why this matters more than Cargo's own per-target locking covers — two
11076/// *different* worktrees building the same package name/version into one
11077/// cache directory is a staleness bug, not a lock contention one.
11078///
11079/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11080/// agent waves do. Repository scheduling also has its own bounded,
11081/// cancellable resource-wait policy. Contention must not consume the
11082/// verification budget: `body` receives the full `budget` after both leases
11083/// are acquired and the cache is fresh. Without a shared cache, only
11084/// repository coordination is acquired before handing over the full budget.
11085///
11086/// A lease that cannot be acquired within the resource wait is reported as a single
11087/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11088/// verification — the same shape a spawn failure already takes in
11089/// [`run_commands`], so a caller need not special-case it.
11090#[allow(clippy::too_many_arguments)]
11091async fn with_cache_lease<'s, F, Fut>(
11092    state: &'s mut RunState,
11093    cache_dir: Option<&Path>,
11094    node: &str,
11095    seat: &str,
11096    worktree: &Path,
11097    head: &str,
11098    budget: Duration,
11099    context: &str,
11100    wait: &crate::repo_lock::WaitPolicy,
11101    body: F,
11102) -> Result<(Vec<CommandOutcome>, bool)>
11103where
11104    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11105    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11106{
11107    // Match editing waves: cache first, repository second. In particular,
11108    // pre_gate must never hold the repository while waiting for a wave's cache.
11109    let started = Instant::now();
11110    let home = crate::run::home();
11111    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11112    let guard = if let Some(cache_dir) = cache_dir {
11113        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11114        match acquire_cache_lease(
11115            state,
11116            cache_dir,
11117            &owner,
11118            resource_limit,
11119            node,
11120            context,
11121            &|| wait.cancelled(),
11122        )
11123        .await
11124        .and_then(|g| {
11125            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11126        }) {
11127            Ok(g) => Some(g),
11128            Err(e) => {
11129                record_cache_error(state, node, &e);
11130                return Ok((
11131                    vec![CommandOutcome {
11132                        command: "(waiting for the shared build cache)".into(),
11133                        code: None,
11134                        output_tail: e.to_string(),
11135                        duration_ms: started.elapsed().as_millis() as u64,
11136                        resource_blocked: true,
11137                    }],
11138                    false,
11139                ));
11140            }
11141        }
11142    } else {
11143        None
11144    };
11145    let _coordination = coordinate_operation(state, worktree, wait).await?;
11146    if let Some(cache_dir) = cache_dir {
11147        let identity = crate::cache::Identity::new(worktree, head);
11148        if let Err(e) = crate::cache::ensure_fresh(
11149            &home,
11150            cache_dir,
11151            &identity,
11152            &state.config.verify.cargo_manifests(),
11153        ) {
11154            record_cache_error(state, node, &e);
11155            state.event(
11156                "verify",
11157                format!(
11158                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11159                    worktree.display(),
11160                    short(head)
11161                ),
11162            );
11163            if let Some(guard) = guard {
11164                guard.release();
11165            }
11166            return Ok((
11167                vec![CommandOutcome {
11168                    command: "(confirming the shared build cache is fresh)".into(),
11169                    code: None,
11170                    output_tail: e.to_string(),
11171                    duration_ms: started.elapsed().as_millis() as u64,
11172                    resource_blocked: true,
11173                }],
11174                false,
11175            ));
11176        }
11177    }
11178    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11179    if !timed_out_pids.is_empty() {
11180        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11181    }
11182    // Drop repository coordination before releasing the cache, reversing the
11183    // acquisition order. A cancelled/error future also drops both guards.
11184    drop(_coordination);
11185    if let Some(guard) = guard {
11186        guard.release();
11187    }
11188    Ok((outcomes, retried))
11189}
11190
11191/// Poll `pids` — commands [`run_commands`] reports as still running when its
11192/// own timeout elapsed — until every one is confirmed gone, or
11193/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11194///
11195/// Real confirmation where confirmation is possible, not a substitute for
11196/// full process-tree observation: a grandchild the timed-out process spawned
11197/// and that survives independently of it is invisible to a pid check the
11198/// same way it always was, and continuing to observe and collect *that*
11199/// stays a different piece of work with its own owner. This only narrows a
11200/// fixed blind wait into an actual check of the pids this process does know
11201/// about.
11202async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11203    wait_for_pids_with(
11204        pids,
11205        crate::proc::pid_alive,
11206        LEASE_RELEASE_POLL,
11207        LEASE_RELEASE_MAX_WAIT,
11208    )
11209    .await;
11210}
11211
11212/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
11213/// interval and ceiling supplied by the caller, so the polling *logic* -
11214/// returns as soon as every pid reports dead, gives up at the ceiling
11215/// otherwise - is testable on millisecond durations without asking the real
11216/// OS about a pid at all.
11217async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11218    pids: &[u32],
11219    alive: F,
11220    poll: Duration,
11221    max_wait: Duration,
11222) {
11223    let deadline = Instant::now() + max_wait;
11224    loop {
11225        if pids.iter().all(|&pid| !alive(pid)) {
11226            return;
11227        }
11228        if Instant::now() >= deadline {
11229            return;
11230        }
11231        tokio::time::sleep(poll).await;
11232    }
11233}
11234
11235/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
11236/// admission that it could not even get a verify command to run, as opposed
11237/// to evidence the command actually produced? A caller that would otherwise
11238/// read a resource-blocked outcome as a red command must check this first:
11239/// see [`Runner::gate`], which retries rather than records `Blocked` when
11240/// this is true.
11241fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11242    outcomes.iter().any(|o| o.resource_blocked)
11243}
11244
11245/// What [`Runner::gate_fix_round`] decided.
11246pub(crate) enum GateFix {
11247    /// The tree changed and `verify.e2e` is still green: run the gate again.
11248    Retry,
11249    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
11250    /// last failure stands and the run ends blocked.
11251    Stop,
11252    /// `verify.e2e` could not run after the fix (magi's own contention):
11253    /// decide nothing now, a later reentry retries.
11254    Defer,
11255}
11256
11257/// Is every red command in `outcomes` an ordinary failure the code could
11258/// explain: it ran, exited non-zero, and said something?
11259///
11260/// A timeout, a spawn failure and a killed process all leave `code` `None`;
11261/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
11262/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
11263/// what the command is stays the gate's business.
11264fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11265    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11266    red.peek().is_some()
11267        && red.all(|o| {
11268            !o.resource_blocked
11269                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11270                && !o.output_tail.trim().is_empty()
11271        })
11272}
11273
11274/// Describe one verify command's outcome for the event log, distinguishing a
11275/// build/link failure — the toolchain never produced a binary to run — from
11276/// an actual test failure, since only the latter is a verdict on the patch.
11277fn e2e_outcome_label(o: &CommandOutcome) -> String {
11278    if o.ok() {
11279        return "pass".to_owned();
11280    }
11281    let reason = if o.build_failed() {
11282        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11283    } else {
11284        format!("FAIL ({:?})", o.code)
11285    };
11286    format!(
11287        "{reason}\n{}",
11288        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11289    )
11290}
11291
11292/// Run `verify.e2e`, retrying once if the first attempt could not build or
11293/// link — a build/link failure is frequently a race against a shared
11294/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
11295/// `verify` event per command, tagged with `context` (normally `"round N"`)
11296/// so the two call sites that need this — the ordinary per-round leg in
11297/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
11298/// it will ever call a round green — read identically in the event log.
11299async fn run_e2e_with_retry(
11300    state: &mut RunState,
11301    shell: &[String],
11302    commands: &[String],
11303    worktree: &Path,
11304    timeout: Duration,
11305    context: &str,
11306) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11307    let (mut e2e, mut timed_out_pids) = run_commands(
11308        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11309    )
11310    .await;
11311    for o in &e2e {
11312        state.event(
11313            "verify",
11314            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11315        );
11316    }
11317    // A build/link failure is not a verdict on the patch — it is frequently a
11318    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
11319    // one retry before letting a red like that decide the round.
11320    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11321    if verify_retried {
11322        state.event(
11323            "verify",
11324            format!(
11325                "{context}: verify could not build/link, not a test result — retrying once \
11326                 before concluding"
11327            ),
11328        );
11329        let retried = run_commands(
11330            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11331        )
11332        .await;
11333        e2e = retried.0;
11334        // Both attempts' timeouts matter, not just the last one: the first
11335        // attempt's descendants may still be alive alongside the retry's.
11336        timed_out_pids.extend(retried.1);
11337        for o in &e2e {
11338            state.event(
11339                "verify",
11340                format!(
11341                    "{context}: retry `{}` -> {}",
11342                    o.command,
11343                    e2e_outcome_label(o)
11344                ),
11345            );
11346        }
11347    }
11348    (e2e, verify_retried, timed_out_pids)
11349}
11350
11351/// Run configured shell commands in `cwd`, in order. The second element is
11352/// the pid of every command that hit `timeout` and was still running when
11353/// this stopped waiting on it (best-effort: `None` when the platform did not
11354/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
11355/// that releases a shared resource afterward needs to know.
11356///
11357/// Records `task` into [`RunState::active`] at every command boundary
11358/// (`RunState::task_command`) and clears it once the whole list has run
11359/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
11360/// for minutes with no seat and no output of its own to show for it (see
11361/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
11362/// apart from "not yet run" without this), and this is the only place that
11363/// knows which command is running right now and how many are left. Three
11364/// saves per command — start, not per second — matching the same "only at a
11365/// boundary" rule [`wave`] already follows for seats.
11366#[allow(clippy::too_many_arguments)]
11367async fn run_commands(
11368    state: &mut RunState,
11369    node: &str,
11370    task: &str,
11371    attempt: usize,
11372    shell: &[String],
11373    commands: &[String],
11374    cwd: &Path,
11375    timeout: Duration,
11376) -> (Vec<CommandOutcome>, Vec<u32>) {
11377    if commands.is_empty() {
11378        // Nothing to mark as running and nothing to clear — an empty list
11379        // means "not configured", and touching `active` (or the disk) over
11380        // that would be a write for every round of a repo with no
11381        // `verify.e2e` / `verify.gate` commands at all.
11382        return (Vec::new(), Vec::new());
11383    }
11384    let mut out = Vec::new();
11385    let mut timed_out_pids = Vec::new();
11386    let total = commands.len();
11387    for (idx, command) in commands.iter().enumerate() {
11388        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11389        if let Err(e) = state.save() {
11390            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11391        }
11392        let started = Instant::now();
11393        let mut cmd = tokio::process::Command::new(&shell[0]);
11394        cmd.quiet();
11395        // Verify the checkout as a standalone caller. Worker ownership and
11396        // pinned executable selection belong to this driver, not to tests or
11397        // tools it runs. Preserve home/seat context and Running's containment.
11398        for key in crate::worker::INVOCATION_ENV {
11399            cmd.env_remove(key);
11400        }
11401        cmd.args(&shell[1..])
11402            .arg(command)
11403            .current_dir(cwd)
11404            .stdin(std::process::Stdio::null())
11405            .stdout(std::process::Stdio::piped())
11406            .stderr(std::process::Stdio::piped())
11407            .kill_on_drop(true);
11408        let mut outputs = Vec::new();
11409        let protected = (|| -> Result<()> {
11410            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
11411                outputs.push(guard);
11412            }
11413            if let Some(cache) = state.config.cache_dir() {
11414                outputs.push(crate::retention::OutputLease::cache(
11415                    &crate::run::home(),
11416                    &cache,
11417                    false,
11418                )?);
11419            }
11420            Ok(())
11421        })();
11422        if let Err(e) = protected {
11423            out.push(CommandOutcome {
11424                command: command.clone(),
11425                code: None,
11426                output_tail: format!("output ownership unavailable: {e:#}"),
11427                duration_ms: 0,
11428                resource_blocked: true,
11429            });
11430            continue;
11431        }
11432        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
11433        let (code, body) = match spawned {
11434            Ok(mut running) => {
11435                let pid = running.child().id();
11436                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
11437                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
11438                let mut paths = vec![cwd.to_owned()];
11439                if let Some(cache) = state.config.cache_dir() {
11440                    paths.push(cache);
11441                }
11442                let mut pressure = None;
11443                let status = tokio::select! {
11444                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
11445                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
11446                        pressure = Some(reason); None
11447                    }
11448                };
11449                let (code, failure) = match status {
11450                    Some(Ok(Ok(status))) => (
11451                        status.code(),
11452                        status
11453                            .code()
11454                            .is_none()
11455                            .then(|| format!("command terminated: {status}")),
11456                    ),
11457                    Some(Ok(Err(error))) => {
11458                        (None, Some(format!("failed waiting for command: {error}")))
11459                    }
11460                    Some(Err(_)) => {
11461                        // Leased cleanup confirms process-tree death itself.
11462                        // Unleased commands still need the caller's PID check
11463                        // after kill-on-drop, before releasing cache ownership.
11464                        if !running.coordinated()
11465                            && let Some(pid) = pid
11466                        {
11467                            timed_out_pids.push(pid);
11468                        }
11469                        (
11470                            None,
11471                            Some(format!("timed out after {}s", timeout.as_secs())),
11472                        )
11473                    }
11474                    None => (None, None),
11475                };
11476                running.finish().await;
11477                if let Some(reason) = pressure {
11478                    state.resource_block = Some(crate::run::ResourceBlock {
11479                        cause: crate::run::ResourceBlockCause::Other,
11480                        node: node.to_owned(),
11481                        kind: "disk pressure".to_owned(),
11482                        detail: reason.clone(),
11483                        at: Timestamp::now(),
11484                    });
11485                    out.push(CommandOutcome {
11486                        command: command.clone(),
11487                        code: None,
11488                        output_tail: reason,
11489                        duration_ms: started.elapsed().as_millis() as u64,
11490                        resource_blocked: true,
11491                    });
11492                    break;
11493                }
11494                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11495                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11496                if let Some(failure) = failure {
11497                    body.push('\n');
11498                    body.push_str(&failure);
11499                }
11500                (code, body)
11501            }
11502            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11503        };
11504        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
11505        if !body.is_empty() {
11506            // The full text is kept whatever the summary dropped. A fresh
11507            // name per command so a resume or retry never overwrites an
11508            // earlier run's output. A failed write changes no verdict.
11509            match crate::run::write_verification_artifact(state, task, &body) {
11510                Ok(name) if summary != body => {
11511                    summary = format!("[full output: artifacts/{name}]\n{summary}");
11512                }
11513                Ok(_) => {}
11514                Err(e) => {
11515                    let warning = format!("could not save the full {task} output: {e:#}");
11516                    tracing::warn!("{warning}");
11517                    state.event(node, warning);
11518                    if summary != body {
11519                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
11520                    }
11521                }
11522            }
11523        }
11524        out.push(CommandOutcome {
11525            command: command.clone(),
11526            code,
11527            output_tail: summary,
11528            duration_ms: started.elapsed().as_millis() as u64,
11529            resource_blocked: false,
11530        });
11531    }
11532    state.task_finished(task);
11533    if let Err(e) = state.save() {
11534        tracing::warn!("could not persist the end of {task}: {e:#}");
11535    }
11536    (out, timed_out_pids)
11537}
11538
11539/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
11540/// section (`report::run`) and in the `merge` event this node records — for
11541/// the operator to run by hand.
11542///
11543/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
11544/// branch whose ruleset forbids merge commits (GitHub's "must not contain
11545/// merge commits", or "require linear history") rejects the push a `--no-ff`
11546/// merge would produce, which is exactly the guidance this function replaces.
11547/// `message`'s first line becomes the squash commit's subject, matching the
11548/// note `report::run` prints alongside this command — see that function for
11549/// why an explicit subject is not optional there.
11550fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11551    let repo = repo.display();
11552    match style {
11553        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11554        MergeStyle::Squash => {
11555            // The subject sits inside double quotes, and a title an agent
11556            // wrote may carry the characters that break out of them.
11557            let subject = message
11558                .lines()
11559                .next()
11560                .unwrap_or(branch)
11561                .replace(['\\', '"', '$', '`'], "");
11562            format!(
11563                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11564            )
11565        }
11566        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11567    }
11568}
11569
11570/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
11571/// under the hood, rejects a `title` over 256 characters and the whole
11572/// command fails — no PR at all, for a run whose body was otherwise fine
11573/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
11574/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
11575/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
11576/// this bound. It is a margin, not a guarantee — a title packed
11577/// with multi-unit characters could still in principle land close to the
11578/// edge, but a real task title's occasional emoji or accented letter fits
11579/// comfortably inside it.
11580const PR_TITLE_MAX: usize = 240;
11581
11582/// A pull request title from the opening line of `text`, or `None` when that
11583/// line is not English (GitHub text is) or has no letters.
11584///
11585/// A line within `max` is kept as is. A longer one is cut at the end of its
11586/// first sentence when that falls inside `max`, else at a word boundary with a
11587/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
11588/// make every merely-truncated title look non-English). The language check
11589/// runs on the kept text before any mark is added, so only what GitHub will
11590/// show is judged: an English opening followed by non-ASCII far past the cut
11591/// still passes.
11592fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11593    let line = queue::first_line(text)?;
11594    let chars: Vec<char> = line.chars().collect();
11595    let (kept, mark) = if chars.len() <= max {
11596        (line.to_owned(), "")
11597    } else if let Some(end) = sentence_end(&chars, max) {
11598        (chars[..end].iter().collect::<String>(), "")
11599    } else {
11600        let room = max.saturating_sub(3);
11601        // Cut at the last space inside the room; when the char just past the
11602        // room is a space the room already ends on a word.
11603        let cut = if chars[room].is_whitespace() {
11604            room
11605        } else {
11606            chars[..room]
11607                .iter()
11608                .rposition(|c| c.is_whitespace())
11609                .unwrap_or(room)
11610        };
11611        let head: String = chars[..cut].iter().collect();
11612        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11613        (head.to_owned(), "...")
11614    };
11615    if kept.is_empty() {
11616        return None;
11617    }
11618    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11619        return None;
11620    }
11621    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11622        return None;
11623    }
11624    Some(format!("{kept}{mark}"))
11625}
11626
11627/// The char length of the first sentence of `chars` when it ends within `max`
11628/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
11629/// abbreviations so `e.g. foo` does not end a title early.
11630fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11631    const MIN: usize = 20;
11632    for i in MIN..max.min(chars.len()) {
11633        if !matches!(chars[i], '.' | '!' | '?') {
11634            continue;
11635        }
11636        let Some(&next) = chars.get(i + 1) else {
11637            continue;
11638        };
11639        if !next.is_whitespace() {
11640            continue;
11641        }
11642        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11643        if after.is_some_and(|c| c.is_ascii_lowercase()) {
11644            continue;
11645        }
11646        let word: String = chars[..i]
11647            .iter()
11648            .rev()
11649            .take_while(|c| !c.is_whitespace())
11650            .collect::<Vec<_>>()
11651            .into_iter()
11652            .rev()
11653            .collect();
11654        let word = word.to_ascii_lowercase();
11655        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11656            continue;
11657        }
11658        let end = chars[..i]
11659            .iter()
11660            .rposition(|c| !c.is_whitespace())
11661            .map_or(i, |p| p + 1);
11662        return Some(end);
11663    }
11664    None
11665}
11666
11667/// What `merge = "pr"` (and the merge commit of the other modes) says about a
11668/// change: a title and a body describing what was *implemented*, not the task
11669/// that asked for it. A task reads as a request; a reader of the merged
11670/// history wants the change.
11671struct PrMessage {
11672    title: String,
11673    body: String,
11674}
11675
11676impl PrMessage {
11677    /// Title, blank line, body. The first line is the squash/merge commit
11678    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
11679    /// has to stay one sensible line.
11680    fn commit_message(&self) -> String {
11681        format!("{}\n\n{}", self.title, self.body)
11682    }
11683}
11684
11685/// The text after a leading `TITLE:` (any case) on `line`.
11686fn title_marker(line: &str) -> Option<&str> {
11687    let line = line.trim();
11688    let head = line.get(..6)?;
11689    head.eq_ignore_ascii_case("title:")
11690        .then(|| line[6..].trim())
11691}
11692
11693/// The implementer's own one-line title: the `TITLE:` line the implement
11694/// prompt asks for at the top of its SUMMARY. Candidate commits are all
11695/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
11696/// source, and a title that says as much is refused here too.
11697fn summary_title(summary: &str) -> Option<String> {
11698    let first = summary.lines().find(|l| !l.trim().is_empty())?;
11699    let raw = title_marker(first)?;
11700    if raw.is_empty() {
11701        return None;
11702    }
11703    let title = queue::title_from(raw, PR_TITLE_MAX);
11704    let lower = title.to_ascii_lowercase();
11705    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11706        return None;
11707    }
11708    Some(title)
11709}
11710
11711/// How `open_review`'s instruction begins; see [`landing_title`].
11712const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11713
11714/// Marker `open_review` gives a candidate that nothing in the roster wrote.
11715pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11716
11717/// Does this run review work that already existed, rather than implement a
11718/// task? Runs recorded before `reviewed_commits` existed carry only the
11719/// candidate marker.
11720pub(crate) fn is_review_run(state: &RunState) -> bool {
11721    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11722}
11723
11724/// The title of a review-only run: the subject of the oldest commit under
11725/// review. Later commits are usually fixups, and `instruction` is the review
11726/// prompt, which says nothing about the change. GitHub text is English, so a
11727/// non-ASCII or blank subject yields `None` and the caller's neutral title.
11728fn review_title(state: &RunState) -> Option<String> {
11729    english_subject(
11730        state.reviewed_commits.as_ref()?.first()?,
11731        state.config.graph.github_english(),
11732    )
11733}
11734
11735/// `raw` as a pull request title, or `None` when it is blank, not English
11736/// (GitHub text is), or one of magi's own candidate commit subjects.
11737fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11738    let raw = raw.trim();
11739    if raw.is_empty() {
11740        return None;
11741    }
11742    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11743        return None;
11744    }
11745    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11746        return None;
11747    }
11748    let title = queue::title_from(raw, PR_TITLE_MAX);
11749    let lower = title.to_ascii_lowercase();
11750    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11751        return None;
11752    }
11753    Some(title)
11754}
11755
11756/// What a review-only run's branch says about itself, read at the moment the
11757/// pull request is opened.
11758#[derive(Debug, Clone, PartialEq, Eq)]
11759struct BranchFacts {
11760    /// `(subject, body)` of each commit, oldest first.
11761    commits: Vec<(String, String)>,
11762    /// Trimmed `git diff --stat`.
11763    stat: String,
11764}
11765
11766/// Longest diff stat shown: this many file lines plus the summary line.
11767const STAT_FILE_LINES: usize = 25;
11768/// Cap on the commit list, well inside GitHub's 65536-character body limit.
11769const COMMITS_MAX_CHARS: usize = 20_000;
11770
11771/// Read the commits and diff stat of `base..branch`. `None` when git cannot
11772/// say or finds nothing, so the caller falls back to what the run recorded.
11773async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11774    let commits = git::commit_log(repo, base, branch).await.ok()?;
11775    if commits.is_empty() {
11776        return None;
11777    }
11778    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11779    let lines: Vec<&str> = stat.lines().collect();
11780    let stat = if lines.len() > STAT_FILE_LINES + 1 {
11781        let omitted = lines.len() - 1 - STAT_FILE_LINES;
11782        let more = format!(" ... {omitted} more file(s)");
11783        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11784        kept.push(&more);
11785        kept.push(lines[lines.len() - 1]);
11786        kept.join("\n")
11787    } else {
11788        lines.join("\n")
11789    };
11790    Some(BranchFacts { commits, stat })
11791}
11792
11793/// Defang what would break the surrounding markdown: a closing `</details>`
11794/// and a code fence.
11795fn markdown_safe(text: &str) -> String {
11796    text.replace("</details>", "&lt;/details&gt;")
11797        .replace("\x60\x60\x60", "~~~")
11798}
11799
11800fn neutral_title(state: &RunState, winner: char) -> String {
11801    format!(
11802        "chore: land candidate {} of run {}",
11803        winner.to_ascii_uppercase(),
11804        state.id
11805    )
11806}
11807
11808/// Reuse the graph's bounded fixer without starting another graph or driver.
11809/// Copy the working state back even on errors so spent rounds remain visible.
11810pub(crate) async fn repair_landing_gate(
11811    state: &mut RunState,
11812    winner: &Candidate,
11813    outcomes: &[CommandOutcome],
11814) -> Result<GateFix> {
11815    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11816        return Ok(GateFix::Stop);
11817    }
11818    let mut runner = Runner {
11819        roles: state.config.resolve_roles()?,
11820        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11821        state: state.clone(),
11822        pause: Pause::new(),
11823        interrupt: Pause::new(),
11824    };
11825    let result = runner.gate_fix_round(winner, outcomes).await;
11826    *state = runner.state;
11827    result
11828}
11829
11830/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
11831/// A completed failure is terminal on that SHA. Resource waits remain retryable.
11832pub(crate) async fn verify_current_head(
11833    state: &mut RunState,
11834    winner: &Candidate,
11835    wait: &crate::repo_lock::WaitPolicy,
11836) -> Result<bool> {
11837    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11838    // Preserve the previous record before replacing it, including on crash recovery.
11839    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
11840        state
11841            .verification_history
11842            .push(crate::run::VerificationEvidence {
11843                superseded_by: Some(head.clone()),
11844                archived_at: Some(Timestamp::now()),
11845                fixes_spent: Some(state.gate_fixes.len()),
11846                gate_head: None,
11847                gate_ran: false,
11848                gate: Vec::new(),
11849                e2e: state.head_e2e.take(),
11850            });
11851    }
11852    if let Some(e) = &state.head_e2e
11853        && e.head == head
11854    {
11855        if e.passed {
11856            return Ok(true);
11857        }
11858        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
11859            state.status = RunStatus::Blocked;
11860            state.save()?;
11861            return Ok(false);
11862        }
11863    } else if state.reviews.last().is_some_and(|r| {
11864        r.verified_head.as_deref() == Some(head.as_str())
11865            && (r.e2e_status() == E2eStatus::Passed
11866                || (r.e2e_status() == E2eStatus::NotConfigured
11867                    && state.config.verify.e2e.is_empty()))
11868    }) || state
11869        .gate_repair_e2e
11870        .as_ref()
11871        .is_some_and(|e| e.head == head && e.passed)
11872    {
11873        return Ok(true);
11874    }
11875    if let Some(e) = &state.gate_repair_e2e
11876        && e.head == head
11877        && !e.passed
11878        && !e.outcomes.is_empty()
11879        && !verify_inconclusive(&e.outcomes)
11880    {
11881        state.status = RunStatus::Blocked;
11882        state.save()?;
11883        return Ok(false);
11884    }
11885    state.head_e2e = Some(crate::run::GateRepairE2e {
11886        head: head.clone(),
11887        passed: false,
11888        outcomes: Vec::new(),
11889    });
11890    state.save()?;
11891    let commands = state.config.verify.e2e.clone();
11892    let shell = state.config.shell();
11893    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11894    let cache_dir = state.config.cache_dir();
11895    let outcomes = if commands.is_empty() {
11896        Vec::new()
11897    } else {
11898        let (outcomes, _) = with_cache_lease(
11899            state,
11900            cache_dir.as_deref(),
11901            "e2e",
11902            "e2e",
11903            &winner.worktree,
11904            &head,
11905            timeout,
11906            "current head e2e",
11907            wait,
11908            |state, budget| async move {
11909                run_e2e_with_retry(
11910                    state,
11911                    &shell,
11912                    &commands,
11913                    &winner.worktree,
11914                    budget,
11915                    "current head e2e",
11916                )
11917                .await
11918            },
11919        )
11920        .await?;
11921        outcomes
11922    };
11923    let unresolved = verify_inconclusive(&outcomes);
11924    let passed = !unresolved
11925        && outcomes.iter().all(CommandOutcome::ok)
11926        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11927    state.head_e2e = Some(crate::run::GateRepairE2e {
11928        head: head.clone(),
11929        passed,
11930        outcomes,
11931    });
11932    if !passed && !unresolved {
11933        state.status = RunStatus::Blocked;
11934        state.event(
11935            "e2e",
11936            format!("verify.e2e failed on head {head}; nothing published"),
11937        );
11938    }
11939    state.save()?;
11940    Ok(passed)
11941}
11942
11943/// Recheck a repair's e2e requirement across failures, resource deferrals and
11944/// process interruptions. A successful result only applies to its exact SHA.
11945pub(crate) async fn check_gate_repair_e2e(
11946    state: &mut RunState,
11947    winner: &Candidate,
11948    wait: &crate::repo_lock::WaitPolicy,
11949) -> Result<GateFix> {
11950    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11951        return Ok(GateFix::Retry);
11952    }
11953    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11954    if state
11955        .gate_repair_e2e
11956        .as_ref()
11957        .is_some_and(|r| r.head == head && r.passed)
11958    {
11959        return Ok(GateFix::Retry);
11960    }
11961    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11962        head: head.clone(),
11963        passed: false,
11964        outcomes: Vec::new(),
11965    });
11966    state.save()?;
11967    let commands = state.config.verify.e2e.clone();
11968    let outcomes = if commands.is_empty() {
11969        Vec::new()
11970    } else {
11971        let shell = state.config.shell();
11972        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11973        let cache_dir = state.config.cache_dir();
11974        let (outcomes, _) = with_cache_lease(
11975            state,
11976            cache_dir.as_deref(),
11977            "e2e",
11978            "e2e",
11979            &winner.worktree,
11980            &head,
11981            timeout,
11982            "gate repair e2e",
11983            wait,
11984            |state, budget| async move {
11985                run_e2e_with_retry(
11986                    state,
11987                    &shell,
11988                    &commands,
11989                    &winner.worktree,
11990                    budget,
11991                    "gate repair e2e",
11992                )
11993                .await
11994            },
11995        )
11996        .await?;
11997        outcomes
11998    };
11999    let deferred = verify_inconclusive(&outcomes);
12000    let passed = outcomes.iter().all(CommandOutcome::ok)
12001        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12002    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12003        head,
12004        passed,
12005        outcomes,
12006    });
12007    state.save()?;
12008    if deferred {
12009        Ok(GateFix::Defer)
12010    } else if !passed {
12011        state.event(
12012            "gate",
12013            "verify.e2e failed after gate repair; nothing published",
12014        );
12015        Ok(GateFix::Stop)
12016    } else {
12017        Ok(GateFix::Retry)
12018    }
12019}
12020
12021/// Run the same configured gate for a landing head and a fresh graph.
12022pub(crate) async fn run_landing_gate(
12023    state: &mut RunState,
12024    winner: &Candidate,
12025    wait: &crate::repo_lock::WaitPolicy,
12026) -> Result<Vec<CommandOutcome>> {
12027    let shell = state.config.shell();
12028    let gate_commands = state.config.verify.gate.clone();
12029    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12030    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12031        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12032        Vec::new()
12033    } else {
12034        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12035        let cache_dir = state.config.cache_dir();
12036        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12037        let (outcomes, _) = with_cache_lease(
12038            state,
12039            cache_dir.as_deref(),
12040            "gate",
12041            "gate",
12042            &winner.worktree,
12043            &head,
12044            timeout,
12045            "final gate",
12046            wait,
12047            |state, budget| {
12048                let shell = shell.clone();
12049                let gate_commands = gate_commands.clone();
12050                let winner = winner.clone();
12051                async move {
12052                    Runner::run_pre_gate(state, &winner).await;
12053                    // Bind command evidence after mechanical fixes, before commands start.
12054                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12055                    let (outcomes, timed_out_pids) = run_commands(
12056                        state,
12057                        "gate",
12058                        "gate",
12059                        0,
12060                        &shell,
12061                        &gate_commands,
12062                        &winner.worktree,
12063                        budget,
12064                    )
12065                    .await;
12066                    (outcomes, false, timed_out_pids)
12067                }
12068            },
12069        )
12070        .await?;
12071        outcomes
12072    };
12073    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12074    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12075        let mut outcomes = outcomes;
12076        outcomes.push(CommandOutcome {
12077            command: "gate head stability".into(), code: None,
12078            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12079            duration_ms: 0, resource_blocked: false,
12080        });
12081        return Ok(outcomes);
12082    }
12083    if outcomes.is_empty() {
12084        // Nothing configured to check — distinct from every other
12085        // silence in this run's event log, since an empty `gate` alone
12086        // no longer says whether the gate ran at all (see
12087        // `RunState::gate_ran`'s own doc).
12088        state.event("gate", "no gate commands configured; not tested");
12089    }
12090    for o in &outcomes {
12091        state.event(
12092            "gate",
12093            format!(
12094                "`{}` -> {}",
12095                o.command,
12096                if o.ok() {
12097                    "pass".to_owned()
12098                } else {
12099                    format!(
12100                        "FAIL ({:?})\n{}",
12101                        o.code,
12102                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12103                    )
12104                }
12105            ),
12106        );
12107    }
12108    Ok(outcomes)
12109}
12110
12111/// The pull request title to hand to `land::merge_subject`. A review-only run
12112/// opened by an earlier build titled its pull request with the review prompt;
12113/// that title is dropped (empty, so the fallback applies) rather than landed.
12114/// Any other title, including an operator's rename, passes through untouched,
12115/// and so does every title of a run that implements a task.
12116pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12117    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12118        ""
12119    } else {
12120        pr_title
12121    }
12122}
12123
12124/// What the squash subject falls back to when the pull request title is empty
12125/// or candidate-shaped: for a review-only run the derived title, never the
12126/// review prompt held in `instruction`.
12127pub fn landing_subject_source(state: &RunState) -> String {
12128    if is_review_run(state) {
12129        let winner = state.candidates.first().map_or('A', |c| c.label);
12130        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12131    }
12132    state.instruction.clone()
12133}
12134
12135/// `summary` without its `TITLE:` line, which the pull request title already
12136/// carries.
12137fn summary_without_title(summary: &str) -> String {
12138    let mut lines = summary.trim().lines().peekable();
12139    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12140        lines.next();
12141    }
12142    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12143}
12144
12145/// The pull request title and body for the winning candidate.
12146///
12147/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12148/// the task's own opening line via [`queue::title_from`] when there is none.
12149/// `state.instruction` can open with blank lines (`task_text` only rejects a
12150/// body that is blank *entirely*), which `title_from` skips.
12151///
12152/// Publishing uses an independent description generated from the final diff
12153/// and structured evidence. Internal summaries and reports remain local.
12154/// Non-publishing previews may display the original task and candidate summary.
12155/// The footer retains the run and candidate provenance.
12156#[cfg(test)]
12157fn pr_message(state: &RunState, winner: char) -> PrMessage {
12158    pr_message_with(state, winner, None)
12159}
12160
12161/// [`pr_message`] with what the branch of a review-only run says about itself.
12162/// `facts` is ignored for a run that implements a task.
12163#[cfg(test)]
12164fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12165    let raw = pr_message_raw(
12166        state,
12167        winner,
12168        facts,
12169        crate::pr_outcome::FinalHead::Unreported,
12170        ChangeNote::NotAsked,
12171    );
12172    let id = crate::scrub::Identity::current();
12173    PrMessage {
12174        title: crate::scrub::scrub(&raw.title, &id),
12175        body: crate::scrub::scrub(&raw.body, &id),
12176    }
12177}
12178
12179/// What the public description may say about the change itself.
12180#[derive(Debug, Clone, Copy)]
12181enum ChangeNote<'a> {
12182    /// A local preview or non-publishing merge mode: no public generation needed.
12183    NotAsked,
12184    /// A summary written from the final diff.
12185    Summary(&'a crate::run::ChangeSummary),
12186    /// A summary was asked for and could not be produced; the text says why.
12187    Unavailable(&'a str),
12188}
12189
12190fn pr_message_raw(
12191    state: &RunState,
12192    winner: char,
12193    facts: Option<&BranchFacts>,
12194    head: crate::pr_outcome::FinalHead<'_>,
12195    change: ChangeNote<'_>,
12196) -> PrMessage {
12197    if !matches!(change, ChangeNote::NotAsked) {
12198        let (title, mut body) = match change {
12199            ChangeNote::Summary(c) => (
12200                c.title.clone(),
12201                format!("## Summary\n\n{}\n\n", c.body.trim()),
12202            ),
12203            ChangeNote::Unavailable(why) => (
12204                neutral_title(state, winner),
12205                format!(
12206                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12207                ),
12208            ),
12209            ChangeNote::NotAsked => unreachable!(),
12210        };
12211        body.push_str(&public_pr_evidence(state, head));
12212        body.push_str(&format!(
12213            "\n---\nmagi:run/{} magi:candidate-{}\n",
12214            state.id,
12215            winner.to_ascii_lowercase()
12216        ));
12217        return PrMessage { title, body };
12218    }
12219    pr_preview(state, winner, facts, head)
12220}
12221
12222/// Append public facts from the run records after every prose generation,
12223/// including posting-gate rewrites. The agent cannot drop this provenance.
12224fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12225    let mut body = crate::pr_outcome::review_rounds(state);
12226    body.push_str(&crate::pr_outcome::final_verification(state, head));
12227    let open = state.last_round_findings();
12228    if !open.is_empty() {
12229        body.push_str("## Open review findings\n\n");
12230        for f in open {
12231            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12232        }
12233        body.push('\n');
12234    }
12235    if let Some(c) = state.change_summary.as_ref()
12236        && head == crate::pr_outcome::FinalHead::At(&c.head)
12237    {
12238        body.push_str(&format!(
12239            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12240            c.base, c.head
12241        ));
12242    }
12243    body
12244}
12245
12246/// Local-only preview of the operator's task and candidate summary.
12247fn pr_preview(
12248    state: &RunState,
12249    winner: char,
12250    facts: Option<&BranchFacts>,
12251    head: crate::pr_outcome::FinalHead<'_>,
12252) -> PrMessage {
12253    let summary = state
12254        .candidates
12255        .iter()
12256        .find(|c| c.label == winner)
12257        .map(|c| c.summary.as_str())
12258        .unwrap_or_default();
12259    // The fallback is the operator's own words and may not be English; GitHub
12260    // text always is, so a non-English task gets a neutral title instead.
12261    let review = is_review_run(state);
12262    let title = if review {
12263        facts
12264            .and_then(|f| {
12265                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12266            })
12267            .or_else(|| review_title(state))
12268            .or_else(|| {
12269                state
12270                    .candidates
12271                    .iter()
12272                    .find(|c| c.label == winner)
12273                    .filter(|c| !c.branch.starts_with("magi/"))
12274                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12275            })
12276            .unwrap_or_else(|| neutral_title(state, winner))
12277    } else {
12278        summary_title(summary).unwrap_or_else(|| {
12279            english_title(
12280                &state.instruction,
12281                PR_TITLE_MAX,
12282                state.config.graph.github_english(),
12283            )
12284            .unwrap_or_else(|| neutral_title(state, winner))
12285        })
12286    };
12287
12288    let mut body = String::new();
12289    let what = summary_without_title(summary);
12290    if !what.is_empty() {
12291        body.push_str("## Summary\n\n");
12292        body.push_str(&what);
12293        body.push_str("\n\n");
12294    }
12295
12296    // Review history uses structured facts across all rounds, never reports.
12297    body.push_str(&crate::pr_outcome::review_rounds(state));
12298    body.push_str(&crate::pr_outcome::final_verification(state, head));
12299    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12300
12301    let open = state.open_findings();
12302    if !open.is_empty() {
12303        body.push_str("## Open review findings\n\n");
12304        for f in &open {
12305            body.push_str(&format!(
12306                "- `{}` [{:?}] {}\n",
12307                f.id, f.severity, "Context retained in local review records."
12308            ));
12309        }
12310        body.push('\n');
12311    }
12312
12313    if let Some(fix) = fix
12314        && !fix.rejected.is_empty()
12315    {
12316        body.push_str("## Declined by the fixer\n\n");
12317        for r in &fix.rejected {
12318            body.push_str(&format!(
12319                "- `{}`: {}\n",
12320                r.id, "Reason retained in local review records."
12321            ));
12322        }
12323        body.push('\n');
12324    }
12325
12326    if review {
12327        // The review prompt is not the task; list what the branch carries.
12328        body.push_str("## Commits under review\n\n");
12329        if let Some(facts) = facts {
12330            let mut left = COMMITS_MAX_CHARS;
12331            for (i, (subject, text)) in facts.commits.iter().enumerate() {
12332                let mut entry = format!("- {}\n", markdown_safe(subject));
12333                for l in markdown_safe(text).lines() {
12334                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
12335                }
12336                if left == 0 {
12337                    body.push_str(&format!(
12338                        "- ... {} more commit(s)\n",
12339                        facts.commits.len() - i
12340                    ));
12341                    break;
12342                }
12343                if entry.len() > left {
12344                    // Even the first commit is cut: one huge body must not
12345                    // push the whole description past GitHub's limit.
12346                    let mut end = left;
12347                    while !entry.is_char_boundary(end) {
12348                        end -= 1;
12349                    }
12350                    entry.truncate(end);
12351                    entry.push_str("\n  ... (truncated)\n");
12352                    left = 0;
12353                } else {
12354                    left -= entry.len();
12355                }
12356                body.push_str(&entry);
12357            }
12358            if !facts.stat.trim().is_empty() {
12359                body.push_str(&format!(
12360                    "\n## Diff stat\n\n```\n{}\n```\n",
12361                    markdown_safe(facts.stat.trim())
12362                ));
12363            }
12364        } else {
12365            match &state.reviewed_commits {
12366                Some(subjects) => {
12367                    for s in subjects {
12368                        body.push_str(&format!("- {}\n", s.trim()));
12369                    }
12370                }
12371                None => {
12372                    // An older run kept only the prompt, with the commit list
12373                    // after its first paragraph.
12374                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
12375                    body.push_str(rest.trim());
12376                    body.push('\n');
12377                }
12378            }
12379        }
12380    } else {
12381        let task = state.instruction.trim();
12382        let task = if task.is_empty() {
12383            "(empty task)"
12384        } else {
12385            task
12386        };
12387        body.push_str(&format!(
12388            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
12389            task.replace("</details>", "&lt;/details&gt;")
12390        ));
12391    }
12392
12393    body.push_str(&format!(
12394        "\n---\nmagi:run/{} magi:candidate-{}\n",
12395        state.id,
12396        winner.to_ascii_lowercase()
12397    ));
12398
12399    PrMessage { title, body }
12400}
12401
12402/// Source imports have already been prepared from structured inputs. Legacy
12403/// inferred references must never instruct a seat to reconstruct those imports.
12404fn seeded_instruction(state: &RunState) -> String {
12405    let mut inputs = Vec::new();
12406    if let Some(start) = &state.sources.start_from {
12407        inputs.push(format!(
12408            "Start from `{}` at `{}` (including ancestry).",
12409            start.token, start.sha
12410        ));
12411    }
12412    for pick in &state.sources.pick_commits {
12413        inputs.push(format!(
12414            "Pick `{}` at `{}` (single commit).",
12415            pick.token, pick.sha
12416        ));
12417    }
12418    if inputs.is_empty() {
12419        state.instruction.clone()
12420    } else {
12421        format!(
12422            "{}\n\n# Explicit source inputs already prepared\n\n{}",
12423            state.instruction,
12424            inputs.join("\n")
12425        )
12426    }
12427}
12428
12429/// Does the winner have no commits ahead of the base it would land on?
12430/// Any failure to find out reads as "not empty": the merge then behaves as it
12431/// always did rather than refusing on a guess.
12432async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
12433    let base = &state.base_branch;
12434    let mut against = base.clone();
12435    if mode == MergeMode::Pr {
12436        // A pull request lands on the remote's base, never the local branch
12437        // of the same name: if that cannot be read, "not empty" is the safe
12438        // answer.
12439        let remote = &state.config.merge.remote;
12440        let tracking = format!("{remote}/{base}");
12441        let fetched = git::fetch(repo, remote, base).await;
12442        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
12443            against = tracking;
12444        } else {
12445            return false;
12446        }
12447    }
12448    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
12449}
12450
12451/// Why nothing was opened for an empty winner, with what the task's own
12452/// references resolved to.
12453fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
12454    let mut detail = format!(
12455        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
12456         nothing to open a pull request for"
12457    );
12458    detail.push_str(
12459        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
12460    );
12461    detail
12462}
12463
12464/// What the `Pr` merge does once it knows whether the branch already has an
12465/// open pull request.
12466#[derive(Debug, PartialEq, Eq)]
12467enum PrPlan {
12468    Create,
12469    Adopt { url: String, title: String },
12470    Stop(String),
12471}
12472
12473/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
12474/// a failed lookup -> stop with the real reason. Never guesses.
12475fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
12476    match found {
12477        Ok(land::OpenPr::None) => PrPlan::Create,
12478        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
12479        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
12480            "several open pull requests exist for this branch, not picking one: {}",
12481            urls.join(" ")
12482        )),
12483        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
12484    }
12485}
12486
12487/// `gh pr create`, returning the PR url.
12488pub(crate) async fn gh_pr_create(
12489    cwd: &Path,
12490    base: &str,
12491    head: &str,
12492    title: &str,
12493    body: &str,
12494) -> Result<String> {
12495    let out = tokio::process::Command::new("gh")
12496        .args([
12497            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
12498        ])
12499        .current_dir(cwd)
12500        .quiet()
12501        .stdin(std::process::Stdio::null())
12502        .output()
12503        .await
12504        .context("spawn gh")?;
12505    if out.status.success() {
12506        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
12507    } else {
12508        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
12509    }
12510}
12511
12512/// Never force deletion of uncommitted work, including unknown legacy trees.
12513async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
12514    match std::fs::symlink_metadata(path) {
12515        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
12516            // A previous cleanup may have removed the tree before a branch
12517            // deletion or state save failed. Unregister it if still present.
12518            git::worktree_remove_clean(repo, path).await?;
12519            return Ok(());
12520        }
12521        Err(e) => return Err(e.into()),
12522        Ok(_) => {}
12523    }
12524    if !git::is_clean(path).await? {
12525        bail!(
12526            "worktree has uncommitted work; refusing removal: {}",
12527            path.display()
12528        );
12529    }
12530    if !git::worktree_remove_clean(repo, path).await? {
12531        bail!("worktree removal refused: {}", path.display());
12532    }
12533    Ok(())
12534}
12535
12536/// Tear a run's worktrees and branches down.
12537///
12538/// `home` is where the updated `run.json` is saved (via
12539/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
12540/// a housekeeping pass already has its own honest `home` handed to it, and
12541/// falling through to the global here would write back through whichever
12542/// directory some other process or test pinned into that `OnceLock` first,
12543/// not the one the caller actually resolved its `runs` and `state` from.
12544pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
12545    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
12546        bail!("refusing fold: {reason}");
12547    }
12548    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
12549        bail!("candidate preservation incomplete; refusing cleanup");
12550    }
12551    // Serialize ordinary folding with the same starts/resumes as target-only
12552    // eviction. Otherwise a second janitor could remove the whole worktree
12553    // around a live output borrower even though target eviction was fenced.
12554    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
12555    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
12556    let _driver = crate::driver::maintenance(home, &state.id)?;
12557    let mut output_guards = Vec::new();
12558    for entry in std::fs::read_dir(state.worktree_root())
12559        .into_iter()
12560        .flatten()
12561        .flatten()
12562    {
12563        if entry.path().join(".git").is_file() {
12564            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
12565        }
12566    }
12567    let repo = state.repo.clone();
12568    // CLI, Web and automatic cleanup all enter here, independently of
12569    // Runner::fold_losers. Protect deletion and shared config release too.
12570    let wait = repository_wait(state, None);
12571    let _coordination = match coordinate_operation(state, &repo, &wait).await {
12572        Ok(lease) => lease,
12573        Err(e) => {
12574            state.save_under(home)?;
12575            return Err(e);
12576        }
12577    };
12578    // Confirmed merges dispose of their owned sources after durable evidence
12579    // capture. Preflight every source before removing anything.
12580    let _source_outputs = if state.status == RunStatus::Merged
12581        && state.merge.as_ref().is_some_and(|merge| merge.ok)
12582    {
12583        verify_merged_fold_sources(state, drop_winner, home).await?
12584    } else {
12585        Vec::new()
12586    };
12587    let root = state.worktree_root();
12588    let winner = state.tally.as_ref().map(|t| t.winner);
12589    let mut removed = Vec::new();
12590
12591    for i in 0..state.candidates.len() {
12592        let c = state.candidates[i].clone();
12593        let is_winner = Some(c.label) == winner;
12594        if is_winner && !drop_winner {
12595            continue;
12596        }
12597        let existed = c.worktree.exists();
12598        // A vanished directory can still have a Git registration that keeps
12599        // its branch checked out. Remove that registration before the branch.
12600        remove_preserved_worktree(&repo, &c.worktree).await?;
12601        if existed {
12602            removed.push(c.worktree.to_string_lossy().into_owned());
12603        }
12604        // A branch handed to a later run (and its pull request) is not this
12605        // run's to delete.
12606        let handed_over = state.released_branches.contains(&c.branch);
12607        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12608            if !git::branch_delete(&repo, &c.branch).await? {
12609                bail!("candidate branch removal refused");
12610            }
12611            removed.push(c.branch.clone());
12612        }
12613        state.candidates[i].folded = true;
12614    }
12615
12616    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12617        let path = name.path();
12618        let keep = !drop_winner
12619            && winner.is_some_and(|w| {
12620                path.file_name()
12621                    .is_some_and(|n| n == format!("cand-{w}").as_str())
12622            });
12623        if keep {
12624            continue;
12625        }
12626        remove_preserved_worktree(&repo, &path).await?;
12627        removed.push(path.to_string_lossy().into_owned());
12628    }
12629
12630    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
12631    // judge worktrees, so once the loop above has cleared all of them out,
12632    // the parent is a bare directory nobody else was ever going to remove -
12633    // git only ever managed what was inside it. Left alone, one of these
12634    // accumulates per fully-folded run; the operator's own machine had 74.
12635    // `remove_if_empty` re-checks rather than assuming: a run whose winner
12636    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
12637    // so does anything a run never claimed that happens to share the bay.
12638    remove_if_empty(&root);
12639
12640    if state.enabled_worktree_config && drop_winner {
12641        // A release, not a raw disable: some sibling run in this repository
12642        // may still hold its own reference (see `git::acquire_worktree_config`),
12643        // and only the last release actually turns the setting back off.
12644        git::release_worktree_config(&repo).await.ok();
12645        state.enabled_worktree_config = false;
12646    }
12647    state.save_under(home)?;
12648    Ok(removed)
12649}
12650
12651async fn verify_merged_fold_sources(
12652    state: &RunState,
12653    drop_winner: bool,
12654    home: &Path,
12655) -> Result<Vec<std::fs::File>> {
12656    let winner = state.tally.as_ref().map(|t| t.winner);
12657    let mut heads = BTreeSet::new();
12658    for candidate in &state.candidates {
12659        if Some(candidate.label) == winner && !drop_winner {
12660            continue;
12661        }
12662        anyhow::ensure!(
12663            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12664            "candidate recovery source is outside the inventoried run root: {}",
12665            candidate.worktree.display()
12666        );
12667        match std::fs::symlink_metadata(&candidate.worktree) {
12668            Ok(_) => {}
12669            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12670            Err(error) => return Err(error.into()),
12671        }
12672        if !state.released_branches.contains(&candidate.branch)
12673            && git::branch_exists(&state.repo, &candidate.branch).await?
12674        {
12675            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12676        }
12677    }
12678    let mut cargo_guards = Vec::new();
12679    let entries = match std::fs::read_dir(state.worktree_root()) {
12680        Ok(entries) => Some(entries),
12681        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12682        Err(error) => return Err(error.into()),
12683    };
12684    for entry in entries.into_iter().flatten() {
12685        let path = entry?.path();
12686        let metadata = std::fs::symlink_metadata(&path)?;
12687        anyhow::ensure!(
12688            metadata.is_dir() && !metadata.file_type().is_symlink(),
12689            "non-directory or symlink recovery source retained: {}",
12690            path.display()
12691        );
12692        if !drop_winner
12693            && winner.is_some_and(|w| {
12694                path.file_name()
12695                    .is_some_and(|name| name == format!("cand-{w}").as_str())
12696            })
12697        {
12698            continue;
12699        }
12700        anyhow::ensure!(
12701            path.join(".git").is_file(),
12702            "unregistered recovery source retained: {}",
12703            path.display()
12704        );
12705        anyhow::ensure!(
12706            git::git(
12707                &path,
12708                &[
12709                    "status",
12710                    "--porcelain",
12711                    "--untracked-files=all",
12712                    "--ignored",
12713                    "--",
12714                    ".",
12715                    ":(exclude)target"
12716                ]
12717            )
12718            .await?
12719            .is_empty(),
12720            "source changes or ignored recovery files retained: {}",
12721            path.display()
12722        );
12723        match std::fs::symlink_metadata(path.join("target")) {
12724            Ok(_) => {
12725                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
12726            }
12727            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12728            Err(error) => return Err(error.into()),
12729        }
12730        for checkpoint in [
12731            "rebase-merge",
12732            "rebase-apply",
12733            "sequencer",
12734            "CHERRY_PICK_HEAD",
12735            "MERGE_HEAD",
12736            "REBASE_HEAD",
12737            "REVERT_HEAD",
12738            "BISECT_LOG",
12739            "AUTO_MERGE",
12740            "MERGE_MSG",
12741        ] {
12742            let location = git::git(
12743                &path,
12744                &[
12745                    "rev-parse",
12746                    "--path-format=absolute",
12747                    "--git-path",
12748                    checkpoint,
12749                ],
12750            )
12751            .await?;
12752            anyhow::ensure!(
12753                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
12754                if e.kind() == std::io::ErrorKind::NotFound),
12755                "Git recovery checkpoint retained: {checkpoint}"
12756            );
12757        }
12758        heads.insert(git::rev_parse(&path, "HEAD").await?);
12759    }
12760    crate::fold_evidence::preserve(state, home, &heads).await?;
12761
12762    Ok(cargo_guards)
12763}
12764
12765/// Remove `dir` if it exists and has nothing in it.
12766///
12767/// Best-effort and silent by design: a directory that is not empty (a run
12768/// whose winner is still parked there, a stray file some other process left)
12769/// is exactly the case this must refuse, and a directory that is already gone
12770/// is not a failure worth reporting either. `std::fs::remove_dir` itself
12771/// already refuses a non-empty directory, so the emptiness check below is
12772/// belt, not suspenders - it is what keeps this from ever attempting the
12773/// removal in the case that matters, rather than trusting `remove_dir`'s
12774/// error path to have no side effects if it ever changed.
12775fn remove_if_empty(dir: &Path) {
12776    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
12777        std::fs::remove_dir(dir).ok();
12778    }
12779}
12780
12781/// Severity of the worst open finding in the last review round, for reporting.
12782pub fn worst_open(state: &RunState) -> Option<Severity> {
12783    state
12784        .reviews
12785        .last()?
12786        .reviews
12787        .iter()
12788        .flat_map(|r| r.findings.iter())
12789        .map(|f| f.severity)
12790        .max()
12791}
12792
12793/// `[worktree] setup` hides its products per worktree, which needs
12794/// `extensions.worktreeConfig`. Taken once per run and released with the run's
12795/// worktrees, like the hook's; every place that runs setup calls this first.
12796async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12797    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12798        git::acquire_worktree_config(repo).await?;
12799        state.enabled_worktree_config = true;
12800    }
12801    Ok(())
12802}
12803
12804#[cfg(test)]
12805mod tests {
12806    #[tokio::test]
12807    async fn launcher_base_fetch_requires_remote_success() {
12808        let tmp = tempfile::tempdir().unwrap();
12809        let repo = tmp.path().join("repo");
12810        let remote = tmp.path().join("remote");
12811        std::fs::create_dir(&repo).unwrap();
12812        std::fs::create_dir(&remote).unwrap();
12813        git::git(&remote, &["init", "--bare", "-b", "main"])
12814            .await
12815            .unwrap();
12816        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12817        git::git(
12818            &repo,
12819            &[
12820                "-c",
12821                "user.name=Fixture",
12822                "-c",
12823                "user.email=fixture@example.invalid",
12824                "commit",
12825                "--allow-empty",
12826                "-m",
12827                "fixture",
12828            ],
12829        )
12830        .await
12831        .unwrap();
12832        git::git(
12833            &repo,
12834            &["remote", "add", "origin", remote.to_str().unwrap()],
12835        )
12836        .await
12837        .unwrap();
12838        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12839        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12840        assert_eq!(
12841            resolve_base(&repo, "main", "origin", None)
12842                .await
12843                .unwrap()
12844                .commit,
12845            head
12846        );
12847        #[cfg(unix)]
12848        {
12849            let policy: crate::config::Coordination = toml::from_str(
12850                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12851            ).unwrap();
12852            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12853            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12854            let held = crate::repo_lock::SeatLease::operation_with(
12855                &repo,
12856                Some(&policy),
12857                &crate::repo_lock::WaitPolicy::default(),
12858            )
12859            .await
12860            .unwrap();
12861            assert!(held.is_some());
12862            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12863            tokio::pin!(fetch);
12864            assert!(
12865                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12866                    .await
12867                    .is_err()
12868            );
12869            assert!(
12870                !repo.join(".git/FETCH_HEAD").exists(),
12871                "no fetch before admission"
12872            );
12873            drop(held);
12874            let fetched = fetch.await.unwrap();
12875            assert_eq!(fetched.commit, head);
12876            assert_eq!(fetched.remote, "origin");
12877            assert_eq!(fetched.branch, "main");
12878        }
12879        // A surviving tracking ref is not successful fetch evidence.
12880        git::git(
12881            &repo,
12882            &[
12883                "remote",
12884                "set-url",
12885                "origin",
12886                tmp.path().join("missing").to_str().unwrap(),
12887            ],
12888        )
12889        .await
12890        .unwrap();
12891        let error = resolve_base(&repo, "main", "origin", None)
12892            .await
12893            .unwrap_err()
12894            .to_string();
12895        assert!(error.contains("cannot read origin/main"));
12896        assert!(error.contains("refusing to branch off"));
12897    }
12898
12899    #[test]
12900    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12901        let leaked = vec!["chore(deps): update a crate".to_owned()];
12902        let own = "fix(daemon): apply a chosen action";
12903        assert!(should_retitle("", own, &leaked));
12904        assert!(should_retitle(
12905            &format!("{REVIEW_PROMPT_OPENING} `x`"),
12906            own,
12907            &leaked
12908        ));
12909        assert!(should_retitle(
12910            "chore: land candidate A of run 1",
12911            own,
12912            &leaked
12913        ));
12914        assert!(should_retitle(
12915            "magi: candidate A (uncommitted work)",
12916            own,
12917            &leaked
12918        ));
12919        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12920        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12921        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12922    }
12923
12924    #[test]
12925    fn pr_merge_plan_creates_adopts_or_stops() {
12926        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12927        assert_eq!(
12928            pr_merge_plan(Ok(land::OpenPr::One {
12929                url: "u".into(),
12930                title: "t".into()
12931            })),
12932            PrPlan::Adopt {
12933                url: "u".into(),
12934                title: "t".into()
12935            }
12936        );
12937        let PrPlan::Stop(many) =
12938            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12939        else {
12940            panic!("many must stop");
12941        };
12942        assert!(many.contains('a') && many.contains('b'));
12943        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12944            panic!("a failed lookup must stop");
12945        };
12946        assert!(err.contains("bad token"));
12947    }
12948
12949    use super::*;
12950    use crate::run::GateStatus;
12951    use std::collections::BTreeMap;
12952    use std::time::Duration;
12953
12954    fn conductor() -> AgentSpec {
12955        AgentSpec {
12956            id: "conductor".to_owned(),
12957            kind: crate::config::AgentKind::Command,
12958            model: None,
12959            command: vec!["true".to_owned()],
12960            extra_args: Vec::new(),
12961            env: BTreeMap::new(),
12962            prompt_delivery: None,
12963            max_turn_secs: None,
12964        }
12965    }
12966
12967    fn spec(id: &str) -> AgentSpec {
12968        AgentSpec {
12969            id: id.to_owned(),
12970            kind: crate::config::AgentKind::Command,
12971            model: None,
12972            command: vec!["true".to_owned()],
12973            extra_args: Vec::new(),
12974            env: BTreeMap::new(),
12975            prompt_delivery: None,
12976            max_turn_secs: None,
12977        }
12978    }
12979
12980    fn ids(xs: &[&str]) -> BTreeSet<String> {
12981        xs.iter().map(|s| (*s).to_owned()).collect()
12982    }
12983
12984    #[test]
12985    fn pick_successor_skips_an_agent_another_seat_holds() {
12986        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
12987        let roster = [spec("a"), spec("b"), spec("c")];
12988        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12989        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12990    }
12991
12992    #[test]
12993    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12994        // The other seat started on c but was handed to d; c is free again.
12995        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12996        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12997        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12998        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12999        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13000    }
13001
13002    #[test]
13003    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13004        let roster = [spec("a"), spec("b")];
13005        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13006        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13007        let carried = ids(&["b"]);
13008        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13009        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13010    }
13011
13012    #[test]
13013    fn pick_successor_returns_none_without_any_untried_successor() {
13014        let roster = [spec("a"), spec("b")];
13015        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13016        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13017        let carried = ids(&["a"]);
13018        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13019    }
13020
13021    #[test]
13022    fn pick_successor_rescue_avoids_another_seats_occupant() {
13023        // b is a carried failure and free; a is held by the other seat.
13024        let roster = [spec("a"), spec("b"), spec("c")];
13025        let carried = ids(&["b", "c"]);
13026        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13027        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13028    }
13029
13030    #[test]
13031    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13032        let roster = [spec("a"), spec("b"), spec("c")];
13033        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13034        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13035    }
13036
13037    #[test]
13038    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13039        let roster = [spec("a"), spec("b"), spec("c")];
13040        let failed = ids(&["a", "b", "c"]);
13041        // Rescue looks at the whole roster, once per id, then runs out.
13042        let mut tried = ids(&["b"]);
13043        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13044        assert_eq!(first.id, "a");
13045        tried.insert(first.id.clone());
13046        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13047        assert_eq!(second.id, "c");
13048        tried.insert(second.id.clone());
13049        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13050    }
13051
13052    #[test]
13053    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13054        let roster = [spec("a"), spec("b")];
13055        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13056        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13057    }
13058
13059    #[test]
13060    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13061        let roster = [spec("a"), spec("b"), spec("c")];
13062        let h = SeatHistory {
13063            failed: ids(&["a"]),
13064            last_ok: Some("b".to_owned()),
13065            last_fail: None,
13066        };
13067        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13068        // A last answerer that left the roster, or later failed, is ignored.
13069        let gone = SeatHistory {
13070            last_ok: Some("zzz".to_owned()),
13071            ..h.clone()
13072        };
13073        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13074        let failed = SeatHistory {
13075            failed: ids(&["a", "b"]),
13076            last_ok: Some("b".to_owned()),
13077            last_fail: None,
13078        };
13079        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13080    }
13081
13082    #[test]
13083    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13084        let first = SeatState::new("review-1", "alpha", 7);
13085        let next = handover_seat("review-1", "gamma", 7);
13086        assert_ne!(first.claude_session, next.claude_session);
13087    }
13088
13089    #[test]
13090    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13091        let mut state = state_with_summary("x", "y");
13092        let a = handover_seat("review-1", "beta", state.next_seat_seed());
13093        let b = handover_seat("review-1", "beta", state.next_seat_seed());
13094        assert_ne!(a.claude_session, b.claude_session);
13095    }
13096
13097    #[test]
13098    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13099        let roster = [spec("a"), spec("b")];
13100        let h = SeatHistory {
13101            failed: ids(&["a", "b"]),
13102            ..SeatHistory::default()
13103        };
13104        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13105        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13106        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13107    }
13108
13109    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
13110    // fallback loop depends on to terminate: it must walk forward from the
13111    // seat's own position, never restart at the front of the roster, and it
13112    // must never hand back an id already tried, however many times that id
13113    // happens to appear.
13114
13115    #[test]
13116    fn failure_signature_ignores_numbers_and_paths() {
13117        assert_eq!(
13118            failure_signature("exited with Some(2) and no usable output"),
13119            failure_signature("exited with Some(137) and no usable output")
13120        );
13121        assert_eq!(
13122            failure_signature("process exited with no usable output (exit code 2)"),
13123            failure_signature("process exited with no usable output (exit code 137)")
13124        );
13125        for (old, new) in [
13126            (
13127                "exited with Some(2) and no usable output",
13128                "process exited with no usable output (exit code 2)",
13129            ),
13130            (
13131                "exited with None and no usable output",
13132                "process exited with no usable output (exit code unavailable)",
13133            ),
13134        ] {
13135            let previous = FailClass::Other(failure_signature(old));
13136            let current = FailClass::Other(failure_signature(new));
13137            assert_eq!(previous, current, "resumption retains the failure class");
13138            assert!(!should_hand_over(Some(&previous), &current));
13139        }
13140        assert_eq!(
13141            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13142            failure_signature("cannot open /var/x.txt: denied")
13143        );
13144        assert_ne!(failure_signature("boom"), failure_signature("bang"));
13145    }
13146
13147    #[test]
13148    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13149        let other = FailClass::Other("x".into());
13150        assert!(should_hand_over(None, &FailClass::Quota));
13151        assert!(should_hand_over(Some(&other), &FailClass::Quota));
13152        assert!(should_hand_over(
13153            Some(&FailClass::Timeout),
13154            &FailClass::Timeout
13155        ));
13156        assert!(should_hand_over(None, &other));
13157        assert!(!should_hand_over(Some(&other), &other));
13158        assert!(should_hand_over(
13159            Some(&other),
13160            &FailClass::Other("y".into())
13161        ));
13162        // A quota or timeout in between ends the run of identical failures.
13163        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13164        assert!(should_hand_over(Some(&FailClass::Quota), &other));
13165    }
13166
13167    #[test]
13168    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13169        let a = SeatState::new("judge-1", "alpha", 7);
13170        let b = handover_seat("judge-1", "beta", 7);
13171        assert_ne!(a.claude_session, b.claude_session);
13172        assert_eq!(b.turns, 0);
13173    }
13174
13175    #[test]
13176    fn a_timeout_is_classified_apart_from_other_failures() {
13177        assert_eq!(
13178            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13179            Some(FailClass::Timeout)
13180        );
13181        assert!(matches!(
13182            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13183            Some(FailClass::Other(_))
13184        ));
13185    }
13186
13187    #[test]
13188    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13189        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13190        let tried = BTreeSet::from(["beta".to_owned()]);
13191        // beta sits at index 1; the next candidate is gamma, never alpha —
13192        // which is very likely a different candidate slot's own agent.
13193        let next = next_untried_in_roster(&roster, 1, &tried);
13194        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13195    }
13196
13197    #[test]
13198    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13199        let roster = vec![spec("alpha"), spec("beta")];
13200        let tried = BTreeSet::from(["beta".to_owned()]);
13201        // beta is the roster's last entry: nothing follows it, and alpha —
13202        // earlier in the roster, almost certainly a different candidate
13203        // slot's own agent — must not be reached by wrapping back to it.
13204        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13205    }
13206
13207    #[test]
13208    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13209        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13210        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13211        // beta (index 1) and gamma (index 2, the only entry after it) have
13212        // both been tried; alpha (index 0) never has, but it comes before
13213        // beta's own position, so there is nothing further for this seat.
13214        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13215    }
13216
13217    #[test]
13218    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13219        let roster = vec![spec("a"), spec("a"), spec("b")];
13220        let tried = BTreeSet::from(["a".to_owned()]);
13221        let next = next_untried_in_roster(&roster, 0, &tried);
13222        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13223    }
13224
13225    #[test]
13226    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13227        let roster = vec![spec("a"), spec("b")];
13228        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13229        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13230    }
13231
13232    #[test]
13233    fn remove_if_empty_only_ever_takes_a_bare_directory() {
13234        let dir = tempfile::tempdir().unwrap();
13235        let bay = dir.path().join("ffff");
13236
13237        // Not there yet: nothing to do, nothing to panic on.
13238        remove_if_empty(&bay);
13239        assert!(!bay.exists());
13240
13241        // Something still inside - the winner's worktree, or a stray file -
13242        // keeps the directory standing.
13243        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13244        remove_if_empty(&bay);
13245        assert!(bay.exists(), "non-empty directory must survive");
13246
13247        // Once the last entry is gone, so is the directory itself.
13248        std::fs::remove_dir(bay.join("cand-A")).unwrap();
13249        remove_if_empty(&bay);
13250        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13251    }
13252
13253    // `round_is_clean` is the exact decision this task fixed: a round with a
13254    // seat that never answered must not read the same as a round every seat
13255    // actually reviewed. These are deterministic and process-free by design —
13256    // the equivalent end-to-end check (a real reviewer timing out under a
13257    // live graph run) is a genuine race against wall-clock contention, and a
13258    // spawn slow enough to blow even a generous budget under a loaded test
13259    // run must not turn this specific regression check flaky.
13260
13261    #[test]
13262    fn a_full_panel_that_found_nothing_is_clean() {
13263        assert!(round_is_clean(
13264            0,
13265            true,
13266            2,
13267            2,
13268            0,
13269            IncompleteReviewPolicy::Block
13270        ));
13271    }
13272
13273    #[test]
13274    fn a_missing_seat_is_never_clean_under_the_default_policy() {
13275        assert!(!round_is_clean(
13276            0,
13277            true,
13278            1,
13279            2,
13280            0,
13281            IncompleteReviewPolicy::Block
13282        ));
13283    }
13284
13285    #[test]
13286    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13287        assert!(!round_is_clean(
13288            1,
13289            true,
13290            1,
13291            2,
13292            0,
13293            IncompleteReviewPolicy::Warn
13294        ));
13295    }
13296
13297    #[test]
13298    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13299        assert!(round_is_clean(
13300            0,
13301            true,
13302            1,
13303            2,
13304            0,
13305            IncompleteReviewPolicy::Warn
13306        ));
13307    }
13308
13309    #[test]
13310    fn a_full_panel_with_an_open_finding_is_not_clean() {
13311        assert!(!round_is_clean(
13312            1,
13313            true,
13314            2,
13315            2,
13316            0,
13317            IncompleteReviewPolicy::Block
13318        ));
13319    }
13320
13321    #[test]
13322    fn a_full_panel_with_a_red_e2e_is_not_clean() {
13323        assert!(!round_is_clean(
13324            0,
13325            false,
13326            2,
13327            2,
13328            0,
13329            IncompleteReviewPolicy::Block
13330        ));
13331    }
13332
13333    // The stall this task closes: under the default `block` policy, a seat
13334    // missing only because it was rate limited must not force a wait for a
13335    // session limit that will not lift by the next round. `round_is_clean`
13336    // is where that quorum carve-out lives; the review loop around it never
13337    // changes what a reviewer's vote or a finding's severity means.
13338
13339    #[test]
13340    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13341        // 1 of 2 answered, and the one missing was quota'd — the exact
13342        // "review-2 rate limited (quota)" shape from the field report.
13343        assert!(round_is_clean(
13344            0,
13345            true,
13346            1,
13347            2,
13348            1,
13349            IncompleteReviewPolicy::Block
13350        ));
13351    }
13352
13353    #[test]
13354    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13355        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
13356        // not a quota loss (`quota_missing` stays 0) — worth another try.
13357        assert!(!round_is_clean(
13358            0,
13359            true,
13360            1,
13361            2,
13362            0,
13363            IncompleteReviewPolicy::Block
13364        ));
13365    }
13366
13367    #[test]
13368    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
13369        assert!(!round_is_clean(
13370            1,
13371            true,
13372            1,
13373            2,
13374            1,
13375            IncompleteReviewPolicy::Block
13376        ));
13377        assert!(!round_is_clean(
13378            0,
13379            false,
13380            1,
13381            2,
13382            1,
13383            IncompleteReviewPolicy::Block
13384        ));
13385    }
13386
13387    #[test]
13388    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
13389        // Every seat quota'd, nobody answered: there is no panel to decide
13390        // on, so this must fall through to the existing block-and-retry
13391        // fallback rather than call an unreviewed patch clean.
13392        assert!(!round_is_clean(
13393            0,
13394            true,
13395            0,
13396            2,
13397            2,
13398            IncompleteReviewPolicy::Block
13399        ));
13400    }
13401
13402    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
13403        CommandOutcome {
13404            command: "test".to_owned(),
13405            code,
13406            output_tail: String::new(),
13407            duration_ms: 0,
13408            resource_blocked,
13409        }
13410    }
13411
13412    #[test]
13413    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
13414        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
13415        assert!(
13416            !verify_inconclusive(&[outcome(Some(1), false)]),
13417            "an ordinary failure is still evidence about the patch"
13418        );
13419        assert!(verify_inconclusive(&[outcome(None, true)]));
13420        assert!(
13421            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
13422            "one inconclusive outcome taints the whole batch"
13423        );
13424        assert!(!verify_inconclusive(&[]));
13425    }
13426
13427    #[tokio::test]
13428    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
13429        // Alive for the first two checks, then dead - confirms the loop
13430        // actually re-polls rather than deciding once and sleeping out the
13431        // ceiling regardless.
13432        let calls = std::sync::atomic::AtomicUsize::new(0);
13433        let started = Instant::now();
13434        wait_for_pids_with(
13435            &[123],
13436            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
13437            Duration::from_millis(5),
13438            Duration::from_secs(5),
13439        )
13440        .await;
13441        assert!(
13442            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
13443            "must keep checking rather than deciding on the first answer"
13444        );
13445        assert!(
13446            started.elapsed() < Duration::from_secs(1),
13447            "must return the moment it is confirmed dead, not wait out the ceiling"
13448        );
13449    }
13450
13451    #[tokio::test]
13452    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
13453        let started = Instant::now();
13454        wait_for_pids_with(
13455            &[123],
13456            |_| true, // never reports dead
13457            Duration::from_millis(5),
13458            Duration::from_millis(30),
13459        )
13460        .await;
13461        let elapsed = started.elapsed();
13462        assert!(
13463            elapsed >= Duration::from_millis(30),
13464            "must not give up before its own ceiling: {elapsed:?}"
13465        );
13466        assert!(
13467            elapsed < Duration::from_secs(1),
13468            "must not wait past its own ceiling either: {elapsed:?}"
13469        );
13470    }
13471
13472    #[tokio::test]
13473    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
13474        let started = Instant::now();
13475        wait_for_pids_with(
13476            &[],
13477            |_| true,
13478            Duration::from_secs(5),
13479            Duration::from_secs(5),
13480        )
13481        .await;
13482        assert!(
13483            started.elapsed() < Duration::from_millis(200),
13484            "an empty pid list has nothing to confirm"
13485        );
13486    }
13487
13488    // `review_conclusion` is the exact decision the review hand-off task
13489    // fixed: a round budget spent (or a tree that stopped moving) must not
13490    // collapse into `Blocked` regardless of what verification actually
13491    // said. Deterministic and process-free for the same reason the
13492    // `round_is_clean` family above is.
13493    fn review_round(
13494        clean: bool,
13495        blocking: usize,
13496        answered: usize,
13497        expected: usize,
13498        progressed: bool,
13499        e2e_ok: bool,
13500    ) -> ReviewRound {
13501        ReviewRound {
13502            round: 1,
13503            head: "h".to_owned(),
13504            verified_head: None,
13505            verified_at: None,
13506            reviews: Vec::new(),
13507            e2e: vec![CommandOutcome {
13508                command: "test".to_owned(),
13509                code: Some(if e2e_ok { 0 } else { 1 }),
13510                output_tail: String::new(),
13511                duration_ms: 0,
13512                resource_blocked: false,
13513            }],
13514            verify_retried: false,
13515            e2e_deferred: false,
13516            e2e_defer_reason: None,
13517            fix: None,
13518            blocking,
13519            answered,
13520            expected,
13521            clean,
13522            progressed,
13523            vote_split: false,
13524            reconsideration: Vec::new(),
13525            verdict: None,
13526        }
13527    }
13528
13529    #[test]
13530    fn review_conclusion_is_none_when_nothing_has_run() {
13531        assert_eq!(review_conclusion(&[], 3), None);
13532    }
13533
13534    #[test]
13535    fn review_conclusion_is_none_while_rounds_remain() {
13536        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
13537        assert_eq!(review_conclusion(&rounds, 3), None);
13538    }
13539
13540    #[test]
13541    fn review_conclusion_is_gating_once_a_round_is_clean() {
13542        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
13543        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
13544    }
13545
13546    #[test]
13547    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
13548        let rounds = vec![
13549            review_round(false, 1, 2, 2, true, true),
13550            review_round(false, 1, 2, 2, true, true),
13551        ];
13552        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
13553    }
13554
13555    #[test]
13556    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
13557        let rounds = vec![
13558            review_round(false, 1, 2, 2, true, true),
13559            review_round(false, 1, 2, 2, true, false),
13560        ];
13561        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
13562    }
13563
13564    #[test]
13565    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
13566        // Magi never got a command to run against this round's own head — a
13567        // resource-blocked attempt, not a red one — so this must never
13568        // settle on `Blocked` the way a genuine e2e failure would. `None`
13569        // here is what tells `Runner::review_loop` to retry the check
13570        // itself rather than trust this cheap recomputation with a verdict
13571        // it cannot actually produce.
13572        let mut blocked = review_round(false, 1, 2, 2, true, false);
13573        blocked.e2e[0].resource_blocked = true;
13574        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
13575        assert_eq!(review_conclusion(&rounds, 2), None);
13576    }
13577
13578    #[test]
13579    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
13580        // Missing input, not a verified tree — never a hand-off candidate.
13581        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
13582        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
13583    }
13584
13585    #[test]
13586    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
13587        let rounds = vec![
13588            review_round(false, 1, 2, 2, false, true),
13589            review_round(false, 1, 2, 2, false, true),
13590        ];
13591        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
13592    }
13593
13594    fn secs(n: u64) -> Duration {
13595        Duration::from_secs(n)
13596    }
13597
13598    /// A throwaway repo with one commit on `main`, for tests that need `merge`
13599    /// to make real (and, if it runs at all, real*ly fail*) git calls.
13600    #[tokio::test]
13601    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13602        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
13603        if std::env::var_os(CHILD).is_none() {
13604            // Other tests reconcile every run in the process-global home,
13605            // briefly taking owner locks. This ancestry test expects an idle
13606            // predecessor, so give it its own process and pinned test home.
13607            let output = std::process::Command::new(std::env::current_exe().unwrap())
13608                .args([
13609                    "--exact",
13610                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
13611                    "--nocapture",
13612                ])
13613                .env(CHILD, "1")
13614                .env_remove("MAGI_WORKER_HOME")
13615                .env_remove("MAGI_WORKER_ID")
13616                .output()
13617                .unwrap();
13618            assert!(
13619                output.status.success()
13620                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
13621                "{}\n{}",
13622                String::from_utf8_lossy(&output.stdout),
13623                String::from_utf8_lossy(&output.stderr)
13624            );
13625            return;
13626        }
13627        ask_test_home();
13628        let tmp = tempfile::tempdir().unwrap();
13629        let repo = tmp.path().join("repo");
13630        std::fs::create_dir_all(&repo).unwrap();
13631        init_repo(&repo);
13632        let g = |args: &[&str]| {
13633            let out = std::process::Command::new("git")
13634                .args(args)
13635                .current_dir(&repo)
13636                .quiet()
13637                .output()
13638                .unwrap();
13639            assert!(
13640                out.status.success(),
13641                "{}",
13642                String::from_utf8_lossy(&out.stderr)
13643            );
13644        };
13645        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13646        g(&["checkout", "-b", "historical"]);
13647        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13648        g(&["add", "."]);
13649        g(&["commit", "-m", "old alternative"]);
13650        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13651        g(&["checkout", "-b", "magi/task-selected", &base]);
13652        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13653        g(&["add", "."]);
13654        g(&["commit", "-m", "measurement infrastructure"]);
13655        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13656        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13657        g(&["add", "."]);
13658        g(&["commit", "-m", "dependent implementation"]);
13659        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13660        g(&["checkout", "main"]);
13661        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13662        g(&["add", "."]);
13663        g(&["commit", "-m", "unrelated base change"]);
13664        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13665        let origin = tmp.path().join("origin.git");
13666        g(&[
13667            "clone",
13668            "--bare",
13669            repo.to_str().unwrap(),
13670            origin.to_str().unwrap(),
13671        ]);
13672        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13673        g(&["fetch", "origin"]);
13674        let selection = crate::pr_recovery::Selection {
13675            url: "https://example.test/project/pull/1".into(),
13676            branch: "magi/task-selected".into(),
13677            head: head.clone(),
13678            source_head: None,
13679            source_run: None,
13680            source_proof: None,
13681            disposition: Some(crate::pr_recovery::Disposition::Review),
13682            instruction: "preserve measured behavior".into(),
13683            approved_at: jiff::Timestamp::now(),
13684        };
13685        let mut template = runner_at(RunStatus::Reviewing);
13686        template.state.repo = repo.clone();
13687        template.state.base_commit = current.clone();
13688        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13689        template.state.origin = Some(crate::run::Origin::queue("selected"));
13690        template.state.pr_head = Some(selection.branch.clone());
13691        template.state.pr_recovery = Some(selection.clone());
13692        let mut runner =
13693            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13694                .await
13695                .unwrap();
13696        let wt = runner.state.winner().unwrap().worktree.clone();
13697        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13698        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13699        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13700        assert!(runner.state.seeds.is_empty());
13701        assert_eq!(runner.state.instruction, "preserve measured behavior");
13702        runner.sync_to_base().await.unwrap();
13703        assert_eq!(
13704            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13705            "prerequisite"
13706        );
13707        assert_eq!(
13708            std::fs::read_to_string(wt.join("implementation")).unwrap(),
13709            "measured implementation"
13710        );
13711        assert_eq!(
13712            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13713            "current main change"
13714        );
13715        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13716        assert_eq!(
13717            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13718            head
13719        );
13720
13721        // A failed selected-source setup stays inspectable, never resumable as a new competition.
13722        let mut partial = runner_at(RunStatus::Reviewing);
13723        partial.state.repo = repo.clone();
13724        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
13725        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
13726            run: Some("exit 1".into()),
13727            ..Default::default()
13728        }];
13729        let selection = runner.state.pr_recovery.clone().unwrap();
13730        partial.state.pr_recovery = Some(selection.clone());
13731        partial.state.pr_head = Some(selection.branch.clone());
13732        partial.state.origin = Some(crate::run::Origin::queue("selected"));
13733        let id = partial.state.id.clone();
13734        let error = match Runner::open_selected(
13735            &repo,
13736            partial.state,
13737            partial.roles,
13738            1,
13739            runner.state.base_commit.clone(),
13740            selection,
13741        )
13742        .await
13743        {
13744            Ok(_) => panic!("setup must fail"),
13745            Err(e) => e,
13746        };
13747        assert_eq!(
13748            error
13749                .downcast_ref::<crate::run::StartupFailure>()
13750                .unwrap()
13751                .run,
13752            id
13753        );
13754        let failed = RunState::load(&id).unwrap();
13755        assert_eq!(failed.status, RunStatus::Failed);
13756        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
13757        assert!(Runner::resume(&id).is_err());
13758        runner.state.status = RunStatus::Ready;
13759        runner.state.driver_exited = true;
13760        runner.state.save().unwrap();
13761        let mut retry = runner_at(RunStatus::Reviewing);
13762        retry.state.repo = repo.clone();
13763        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
13764        let mut reselected = runner.state.pr_recovery.clone().unwrap();
13765        reselected.approved_at = jiff::Timestamp::now();
13766        retry.state.origin = Some(crate::run::Origin::queue("selected"));
13767        retry.state.pr_head = Some(reselected.branch.clone());
13768        retry.state.pr_recovery = Some(reselected.clone());
13769        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
13770            .await
13771            .unwrap();
13772        let restarted = Runner::open_selected(
13773            &repo,
13774            retry.state,
13775            retry.roles,
13776            1,
13777            runner.state.base_commit.clone(),
13778            reselected,
13779        )
13780        .await
13781        .unwrap();
13782        assert_eq!(
13783            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
13784                .await
13785                .unwrap(),
13786            head
13787        );
13788        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13789        assert_eq!(
13790            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13791            head
13792        );
13793    }
13794
13795    #[tokio::test]
13796    #[cfg(unix)]
13797    async fn fallback_and_interrupted_restoration_require_coordination() {
13798        use std::os::fd::AsRawFd as _;
13799        ask_test_home();
13800        let tmp = tempfile::tempdir().unwrap();
13801        let mut runner = registered_runner(tmp.path()).await;
13802        runner.state.config.graph.timeout_resource_wait = 1;
13803        let winner = runner.state.candidates[0].clone();
13804        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13805        let leftover = winner.worktree.join("leftover");
13806        std::fs::write(&leftover, "untrusted edits").unwrap();
13807        let next = tmp.path().join("next-fixer");
13808        let mut failed = spec("first");
13809        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
13810        let mut successor = spec("second");
13811        successor.command = vec![
13812            "sh".into(),
13813            "-c".into(),
13814            format!("touch '{}'; echo done", next.display()),
13815        ];
13816        let attempts = vec![
13817            (failed.clone(), "first".into()),
13818            (successor.clone(), "second".into()),
13819        ];
13820        let lock_path = runner.state.repo.join(".git/updater.lock");
13821        let external = std::fs::File::create(&lock_path).unwrap();
13822        assert_eq!(
13823            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13824            0
13825        );
13826        let artifacts = tmp.path().join("artifacts");
13827        let build = |spec, seat| SeatJob {
13828            spec,
13829            seat,
13830            cwd: winner.worktree.clone(),
13831            prompt: "fixture".into(),
13832            timeout: Duration::from_secs(10),
13833            allow_write: true,
13834            sessions: false,
13835            artifacts: artifacts.clone(),
13836            stem: "fix".into(),
13837            handover: None,
13838        };
13839        let (_, _, out) = runner
13840            .ask_fixer_among(
13841                attempts.clone(),
13842                "graft",
13843                None,
13844                Some((&winner.worktree, &head)),
13845                build,
13846            )
13847            .await;
13848        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13849        assert!(leftover.exists(), "contention must prevent clean");
13850        assert!(!next.exists());
13851        assert!(runner.state.handovers.is_empty());
13852        // Early interrupted-repair rollback uses the same protected route.
13853        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13854            from_run: "earlier".into(),
13855            generation: 1,
13856            mode: crate::recovery::Mode::VerifyRepair,
13857            reasons: Vec::new(),
13858            reviewed_head: None,
13859            rounds: Vec::new(),
13860            open_findings: Vec::new(),
13861            evidence: None,
13862            criteria_inherited: true,
13863            repair: crate::recovery::Repair::Started {
13864                before: head.clone(),
13865            },
13866        });
13867        assert!(
13868            runner
13869                .recovery_repair(&winner)
13870                .await
13871                .unwrap_err()
13872                .is::<crate::repo_lock::ResourceBlocked>()
13873        );
13874        assert!(matches!(
13875            runner.state.recovery.as_ref().unwrap().repair,
13876            crate::recovery::Repair::Started { .. }
13877        ));
13878        assert!(leftover.exists());
13879        drop(external);
13880        runner.recovery_repair(&winner).await.unwrap();
13881        assert!(!leftover.exists());
13882        assert!(matches!(
13883            runner.state.recovery.as_ref().unwrap().repair,
13884            crate::recovery::Repair::Done { .. }
13885        ));
13886        // A hard lock-open failure also leaves HEAD and untracked work intact.
13887        std::fs::remove_file(&lock_path).unwrap();
13888        std::fs::create_dir(&lock_path).unwrap();
13889        std::fs::write(&leftover, "untrusted edits").unwrap();
13890        let (_, _, out) = runner
13891            .ask_fixer_among(
13892                attempts,
13893                "graft",
13894                None,
13895                Some((&winner.worktree, &head)),
13896                build,
13897            )
13898            .await;
13899        assert!(matches!(out, AgentOutcome::Failed(_)));
13900        assert!(leftover.exists());
13901        assert!(!next.exists());
13902        assert_eq!(
13903            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13904            head
13905        );
13906    }
13907
13908    #[cfg(unix)]
13909    async fn registered_runner(root: &Path) -> Runner {
13910        repository_runner(root, crate::config::test_coordination()).await
13911    }
13912
13913    async fn repository_runner(
13914        root: &Path,
13915        coordination: Option<crate::config::Coordination>,
13916    ) -> Runner {
13917        ask_test_home();
13918        let repo = root.join("repo");
13919        std::fs::create_dir_all(&repo).unwrap();
13920        init_repo(&repo);
13921        if coordination.is_some() {
13922            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13923        }
13924        let mut runner = runner_at(RunStatus::Prep);
13925        runner.state.repo = repo.clone();
13926        runner.state.config.coordination = coordination;
13927        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13928        runner.state.config.graph.worktree_root = Some(root.join("wt"));
13929        runner.state.config.graph.advise = false;
13930        runner.roles.implementers = vec![spec("implement")];
13931        runner.roles.judges = vec![spec("judge")];
13932        runner.prep().await.unwrap();
13933        runner
13934    }
13935
13936    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
13937
13938    #[tokio::test]
13939    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
13940        let tmp = tempfile::tempdir().unwrap();
13941        let mut runner = repository_runner(tmp.path(), None).await;
13942        let prompts = runner.state.config.prompts.clone();
13943        let run = runner.state.id.clone();
13944        let ctx = WaveCtx {
13945            carry_seats: false,
13946            pauses: None,
13947            run: &run,
13948            node: "advise",
13949            prompts: &prompts,
13950            cache: None,
13951            round: None,
13952        };
13953        let jobs: Vec<_> = ["alpha", "beta"]
13954            .into_iter()
13955            .map(|id| SeatJob {
13956                spec: scripted(id, AUTH_OUTAGE),
13957                seat: SeatState::new(id, id, 7),
13958                prompt: "propose".into(),
13959                cwd: runner.state.candidates[0].worktree.clone(),
13960                timeout: Duration::from_secs(10),
13961                allow_write: false,
13962                sessions: false,
13963                artifacts: agent::artifacts_dir(&runner.state.dir()),
13964                stem: id.into(),
13965                handover: None,
13966            })
13967            .collect();
13968        let mut losses = Vec::new();
13969        let results = ask_wave_with(
13970            jobs.clone(),
13971            runner.sem.clone(),
13972            2,
13973            &[],
13974            &ctx,
13975            &mut losses,
13976            &mut runner.state,
13977            &|_: &str| Ok(()),
13978        )
13979        .await;
13980        assert!(
13981            results
13982                .iter()
13983                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
13984        );
13985        assert_eq!(runner.state.quota.len(), 2);
13986        assert!(
13987            runner
13988                .state
13989                .quota
13990                .iter()
13991                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
13992        );
13993        assert!(!runner.state.advise_attempted);
13994        runner.stop_for_resource().unwrap();
13995        assert_eq!(runner.state.status, RunStatus::Blocked);
13996        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
13997        assert_eq!(
13998            runner.state.resource_block.as_ref().unwrap().kind,
13999            "agent authentication"
14000        );
14001        // A machine loss followed by an ordinary failure is not evidence
14002        // that the whole node was unavailable, and must not earn a refund.
14003        runner.state.resource_block = None;
14004        runner.state.quota.clear();
14005        let mut mixed = jobs;
14006        mixed[1].spec.command = vec!["false".into()];
14007        let results = ask_wave_with(
14008            mixed,
14009            runner.sem.clone(),
14010            0,
14011            &[],
14012            &ctx,
14013            &mut losses,
14014            &mut runner.state,
14015            &|_: &str| Ok(()),
14016        )
14017        .await;
14018        assert!(results.iter().all(|(_, res, _)| res.is_err()));
14019        assert!(runner.state.resource_block.is_none());
14020        assert_eq!(losses.len(), 1);
14021    }
14022
14023    #[tokio::test]
14024    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14025        let tmp = tempfile::tempdir().unwrap();
14026        let mut runner = repository_runner(tmp.path(), None).await;
14027        let implementer = scripted("implement", AUTH_OUTAGE);
14028        runner.roles.implementers = vec![implementer.clone()];
14029        runner.state.config.agents = vec![implementer];
14030        runner.implement().await.unwrap();
14031        assert_eq!(runner.state.status, RunStatus::Blocked);
14032        assert_eq!(runner.state.quota.len(), 1);
14033        assert_eq!(
14034            runner.state.quota[0].cause,
14035            agent::QuotaCause::Authentication
14036        );
14037        let candidate = &runner.state.candidates[0];
14038        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14039        assert_eq!(runner.state.seats["impl-A"].turns, 1);
14040    }
14041
14042    #[tokio::test]
14043    #[cfg(unix)]
14044    async fn parked_implementer_persists_completed_work() {
14045        for registered in [false, true] {
14046            let tmp = tempfile::tempdir().unwrap();
14047            let mut runner = registered_runner(tmp.path()).await;
14048            if !registered {
14049                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14050            }
14051            let marker = tmp.path().join("started");
14052            let release = tmp.path().join("release");
14053            let mut implementer = spec("implement");
14054            implementer.command = vec![
14055                "sh".into(),
14056                "-c".into(),
14057                format!(
14058                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14059                    marker.display(),
14060                    release.display()
14061                ),
14062            ];
14063            runner.roles.implementers = vec![implementer.clone()];
14064            runner.state.config.agents = vec![implementer];
14065            let pause = runner.pause.clone();
14066            let (result, ()) = tokio::join!(runner.implement(), async {
14067                tokio::time::timeout(Duration::from_secs(10), async {
14068                    while !marker.exists() {
14069                        tokio::time::sleep(Duration::from_millis(10)).await;
14070                    }
14071                })
14072                .await
14073                .unwrap();
14074                pause.park();
14075                std::fs::write(&release, "go").unwrap();
14076            });
14077            result.unwrap();
14078            let candidate = &runner.state.candidates[0];
14079            assert!(candidate.commits > 0, "{candidate:?}");
14080            assert!(candidate.summary.contains("saved-result"));
14081            assert_eq!(candidate.files, 1);
14082            let before = candidate.commits;
14083            runner.implement().await.unwrap();
14084            assert_eq!(runner.state.candidates[0].commits, before);
14085        }
14086    }
14087
14088    #[tokio::test]
14089    #[cfg(unix)]
14090    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14091        let tmp = tempfile::tempdir().unwrap();
14092        let mut runner = registered_runner(tmp.path()).await;
14093        let winner = runner.state.candidates[0].clone();
14094        let cache = tmp.path().join("cache");
14095        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14096        runner.state.config.graph.timeout_resource_wait = 0;
14097        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14098        let crate::cache::AcquireOutcome::Acquired(held) =
14099            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14100        else {
14101            panic!("cache must be free")
14102        };
14103        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14104            from_run: "earlier".into(),
14105            generation: 1,
14106            mode: crate::recovery::Mode::VerifyRepair,
14107            reasons: Vec::new(),
14108            reviewed_head: None,
14109            rounds: Vec::new(),
14110            open_findings: Vec::new(),
14111            evidence: None,
14112            criteria_inherited: true,
14113            repair: crate::recovery::Repair::NotStarted,
14114        });
14115        runner.recovery_repair(&winner).await.unwrap();
14116        assert!(matches!(
14117            runner.state.recovery.as_ref().unwrap().repair,
14118            crate::recovery::Repair::WaitingForRestore { .. }
14119        ));
14120        drop(held);
14121        runner.state.resource_block = None;
14122        runner.state.config.graph.timeout_resource_wait = 1;
14123        runner.recovery_repair(&winner).await.unwrap();
14124        assert!(matches!(
14125            runner.state.recovery.as_ref().unwrap().repair,
14126            crate::recovery::Repair::Done { .. }
14127        ));
14128        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14129    }
14130
14131    #[tokio::test]
14132    #[cfg(unix)]
14133    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14134        use std::os::fd::AsRawFd as _;
14135        for graft in [false, true] {
14136            let tmp = tempfile::tempdir().unwrap();
14137            let mut runner = registered_runner(tmp.path()).await;
14138            runner.state.config.graph.timeout_resource_wait = 1;
14139            let winner = runner.state.candidates[0].clone();
14140            let marker = tmp.path().join("fixer-started");
14141            let mut fixer = spec("fixer");
14142            fixer.command = vec![
14143                "sh".into(),
14144                "-c".into(),
14145                format!("echo started >> '{}'; echo done", marker.display()),
14146            ];
14147            runner.roles.fixer = Some(vec![fixer]);
14148            if graft {
14149                let mut loser = winner.clone();
14150                loser.label = 'B';
14151                loser.index = 1;
14152                runner.state.candidates.push(loser);
14153                runner.state.config.graph.graft_runner_up = true;
14154                runner.state.tally = Some(Tally {
14155                    first_choice: BTreeMap::new(),
14156                    borda: BTreeMap::new(),
14157                    winner: winner.label,
14158                    rankings: 1,
14159                    unanimous_initial: true,
14160                    deliberated: false,
14161                    changed_votes: 0,
14162                    unanimous_final: true,
14163                    tie_break: None,
14164                    judges: 1,
14165                    present: 1,
14166                    quorum: 1,
14167                    met_quorum: true,
14168                    uncontested: None,
14169                });
14170                runner.state.judgements.push(crate::run::Judgement {
14171                    judge: 1,
14172                    seat: "judge-1".into(),
14173                    agent: "judge".into(),
14174                    ranking: vec![winner.label, 'B'],
14175                    reasons: BTreeMap::new(),
14176                    confidence: None,
14177                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14178                    order: vec![0, 1],
14179                    failed: None,
14180                    duration_ms: 0,
14181                });
14182            } else {
14183                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14184                    from_run: "earlier".into(),
14185                    generation: 1,
14186                    mode: crate::recovery::Mode::VerifyRepair,
14187                    reasons: Vec::new(),
14188                    reviewed_head: None,
14189                    rounds: Vec::new(),
14190                    open_findings: Vec::new(),
14191                    evidence: None,
14192                    criteria_inherited: true,
14193                    repair: crate::recovery::Repair::NotStarted,
14194                });
14195            }
14196            let reader = crate::repo_lock::SeatLease::for_checkout(
14197                &winner.worktree,
14198                false,
14199                crate::config::test_coordination().as_ref(),
14200            )
14201            .await
14202            .unwrap();
14203            let pause = runner.pause.clone();
14204            let (result, ()) = tokio::join!(
14205                async {
14206                    if graft {
14207                        runner.graft().await
14208                    } else {
14209                        runner.recovery_repair(&winner).await
14210                    }
14211                },
14212                async {
14213                    tokio::time::sleep(Duration::from_millis(100)).await;
14214                    pause.park();
14215                }
14216            );
14217            result.unwrap();
14218            assert!(!marker.exists(), "a blocked seat must not spawn");
14219            // Exercise the persisted state, as a binary replacement/resume does.
14220            runner.state = RunState::load(&runner.state.id).unwrap();
14221            if graft {
14222                assert_eq!(
14223                    runner.state.graft.as_ref().unwrap().outcome,
14224                    GraftOutcome::WaitingForRestore
14225                );
14226            } else {
14227                assert!(matches!(
14228                    runner.state.recovery.as_ref().unwrap().repair,
14229                    crate::recovery::Repair::WaitingForRestore { .. }
14230                ));
14231            }
14232            drop(reader);
14233            runner.pause = Pause::new();
14234            runner.state.resource_block = None;
14235            // Deferred restoration must still respect an external lock owner.
14236            let external =
14237                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14238            assert_eq!(
14239                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14240                0
14241            );
14242            let leftover = winner.worktree.join("leftover");
14243            std::fs::write(&leftover, "earlier fallback edits").unwrap();
14244            let result = if graft {
14245                runner.graft().await
14246            } else {
14247                runner.recovery_repair(&winner).await
14248            };
14249            assert!(
14250                result
14251                    .unwrap_err()
14252                    .is::<crate::repo_lock::ResourceBlocked>()
14253            );
14254            assert!(leftover.exists());
14255            assert!(!marker.exists());
14256            drop(external);
14257            runner.state.resource_block = None;
14258            let result = if graft {
14259                runner.graft().await
14260            } else {
14261                runner.recovery_repair(&winner).await
14262            };
14263            result.unwrap();
14264            assert!(!leftover.exists(), "restore must precede retry");
14265            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14266            if graft {
14267                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14268            } else {
14269                assert!(matches!(
14270                    runner.state.recovery.as_ref().unwrap().repair,
14271                    crate::recovery::Repair::Done { .. }
14272                ));
14273            }
14274        }
14275    }
14276
14277    #[tokio::test]
14278    #[cfg(unix)]
14279    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
14280        use std::os::fd::AsRawFd as _;
14281        let tmp = tempfile::tempdir().unwrap();
14282        let mut runner = registered_runner(tmp.path()).await;
14283        runner.state.config.graph.timeout_resource_wait = 1;
14284        let winner = runner.state.candidates[0].clone();
14285        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14286        runner.state.tally = Some(Tally {
14287            first_choice: BTreeMap::new(),
14288            borda: BTreeMap::new(),
14289            winner: winner.label,
14290            rankings: 0,
14291            unanimous_initial: false,
14292            deliberated: false,
14293            changed_votes: 0,
14294            unanimous_final: false,
14295            tie_break: None,
14296            judges: 0,
14297            present: 0,
14298            quorum: 0,
14299            met_quorum: true,
14300            uncontested: Some("fixture".into()),
14301        });
14302        runner.state.graft = Some(GraftRecord {
14303            pre: pre.clone(),
14304            post: None,
14305            lines: None,
14306            limit: None,
14307            agent: None,
14308            outcome: GraftOutcome::Running,
14309            requested: Vec::new(),
14310            adoption: None,
14311            adoption_error: None,
14312        });
14313        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
14314        git::commit_all(&winner.worktree, "unfinished graft")
14315            .await
14316            .unwrap();
14317        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14318        let leftover = winner.worktree.join("leftover");
14319        std::fs::write(&leftover, "untrusted edits").unwrap();
14320        let path = runner.state.repo.join(".git/updater.lock");
14321        let external = std::fs::File::create(&path).unwrap();
14322        assert_eq!(
14323            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14324            0
14325        );
14326        assert!(
14327            runner
14328                .graft()
14329                .await
14330                .unwrap_err()
14331                .is::<crate::repo_lock::ResourceBlocked>()
14332        );
14333        assert!(matches!(
14334            runner.state.graft.as_ref().unwrap().outcome,
14335            GraftOutcome::Running
14336        ));
14337        assert_eq!(
14338            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14339            unfinished
14340        );
14341        assert!(leftover.exists());
14342        drop(external);
14343        std::fs::remove_file(&path).unwrap();
14344        std::fs::create_dir(&path).unwrap();
14345        assert!(runner.graft().await.is_err());
14346        assert!(matches!(
14347            runner.state.graft.as_ref().unwrap().outcome,
14348            GraftOutcome::Running
14349        ));
14350        assert_eq!(
14351            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14352            unfinished
14353        );
14354        assert!(leftover.exists());
14355        std::fs::remove_dir(&path).unwrap();
14356        runner.graft().await.unwrap();
14357        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14358        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
14359        assert!(!leftover.exists());
14360    }
14361
14362    #[tokio::test]
14363    #[cfg(unix)]
14364    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
14365        use std::os::fd::AsRawFd as _;
14366        let tmp = tempfile::tempdir().unwrap();
14367        let mut runner = registered_runner(tmp.path()).await;
14368        runner.state.prep_complete = Some(false);
14369        runner.state.config.graph.timeout_resource_wait = 1;
14370        runner.state.save().unwrap();
14371        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14372        assert_eq!(
14373            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14374            0
14375        );
14376        runner.execute().await.unwrap();
14377        let disk = RunState::load(&runner.state.id).unwrap();
14378        assert!(disk.driver_exited);
14379        assert_eq!(disk.driver_pid, None);
14380        assert_eq!(runner.state.status, RunStatus::Blocked);
14381        assert_eq!(
14382            runner.state.resource_block.as_ref().unwrap().kind,
14383            "repository coordination"
14384        );
14385        assert!(runner.state.prep_failure.is_none());
14386        assert!(runner.state.seats.is_empty());
14387        runner.pause.park();
14388        // Include headroom for Git subprocesses under parallel test load;
14389        // the repository contention deadline itself remains one second.
14390        tokio::time::timeout(Duration::from_secs(10), runner.execute())
14391            .await
14392            .unwrap()
14393            .unwrap();
14394        assert!(runner.state.parked);
14395        assert!(runner.state.prep_failure.is_none());
14396        let disk = RunState::load(&runner.state.id).unwrap();
14397        assert!(disk.driver_exited);
14398        assert_eq!(disk.driver_pid, None);
14399    }
14400
14401    #[tokio::test]
14402    #[cfg(unix)]
14403    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
14404        use std::os::fd::AsRawFd as _;
14405        let tmp = tempfile::tempdir().unwrap();
14406        let mut runner = registered_runner(tmp.path()).await;
14407        runner.state.prep_complete = Some(false);
14408        runner.state.config.graph.timeout_resource_wait = 30;
14409        runner.state.save().unwrap();
14410        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14411        assert_eq!(
14412            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14413            0
14414        );
14415        assert!(
14416            tokio::time::timeout(Duration::from_millis(100), runner.execute())
14417                .await
14418                .is_err()
14419        );
14420        let disk = RunState::load(&runner.state.id).unwrap();
14421        assert!(disk.driver_exited);
14422        assert_eq!(disk.driver_pid, None);
14423        assert!(disk.status.resumable());
14424        assert!(runner.state.driver_exited);
14425        assert_eq!(runner.state.driver_pid, None);
14426        runner.state.save().unwrap();
14427        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
14428    }
14429
14430    #[tokio::test]
14431    #[cfg(unix)]
14432    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
14433        let tmp = tempfile::tempdir().unwrap();
14434        let mut runner = registered_runner(tmp.path()).await;
14435        let winner = runner.state.candidates[0].clone();
14436        let cache = tmp.path().join("cache");
14437        let started = tmp.path().join("pre-gate-started");
14438        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
14439        runner.state.config.verify.gate =
14440            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
14441        let home = crate::run::home();
14442        let owner =
14443            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
14444        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
14445            crate::cache::AcquireOutcome::Acquired(g) => g,
14446            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
14447        };
14448        let mut gate = Box::pin(runner.run_gate(&winner));
14449        assert!(
14450            tokio::time::timeout(Duration::from_millis(150), &mut gate)
14451                .await
14452                .is_err()
14453        );
14454        assert!(!started.exists(), "pre-gate must wait for the cache too");
14455        // Simulate the wave that already owns the cache entering invoke.
14456        // It must obtain the editor reservation and finish without waiting
14457        // for the gate that is queued behind its cache.
14458        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
14459        let editor = crate::repo_lock::SeatLease::checkout_with(
14460            &winner.worktree,
14461            true,
14462            crate::config::test_coordination().as_ref(),
14463            &wait,
14464        )
14465        .await
14466        .unwrap();
14467        drop(editor);
14468        guard.release();
14469        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14470            .await
14471            .unwrap()
14472            .unwrap();
14473        drop(gate);
14474        assert!(outcomes.iter().all(CommandOutcome::ok));
14475        assert!(started.exists());
14476    }
14477
14478    #[tokio::test]
14479    #[cfg(unix)]
14480    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
14481        use std::os::fd::AsRawFd as _;
14482
14483        ask_test_home();
14484        let tmp = tempfile::tempdir().unwrap();
14485        let repo = tmp.path().join("repo");
14486        std::fs::create_dir_all(&repo).unwrap();
14487        init_repo(&repo);
14488        let common = repo.join(".git");
14489        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14490        let mut runner = runner_at(RunStatus::Prep);
14491        runner.state.repo = repo.clone();
14492        runner.state.config.coordination = crate::config::test_coordination();
14493        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14494        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14495        runner.state.config.graph.advise = false;
14496        runner.roles.implementers = vec![spec("implement")];
14497        runner.roles.judges = vec![spec("judge")];
14498        runner.prep().await.unwrap();
14499        let winner = runner.state.candidates[0].clone();
14500        // Emulate edits already left by a completed seat. The fixture CLI
14501        // only reports completion; it performs no repository operations.
14502        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
14503        let completed = tmp.path().join("seat-completed");
14504        runner.roles.implementers[0].command = vec![
14505            "sh".into(),
14506            "-c".into(),
14507            format!("touch '{}'; echo completed", completed.display()),
14508        ];
14509        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
14510        assert_eq!(
14511            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14512            0
14513        );
14514        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14515        let mut implement = Box::pin(runner.implement());
14516        tokio::select! {
14517            result = &mut implement => panic!("rescue must wait: {result:?}"),
14518            result = tokio::time::timeout(Duration::from_secs(10), async {
14519                while !completed.exists() {
14520                    tokio::time::sleep(Duration::from_millis(20)).await;
14521                }
14522            }) => result.unwrap(),
14523        }
14524        assert!(
14525            tokio::time::timeout(Duration::from_millis(200), &mut implement)
14526                .await
14527                .is_err()
14528        );
14529        assert_eq!(
14530            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14531            before
14532        );
14533        assert_eq!(
14534            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14535            0
14536        );
14537        tokio::time::timeout(Duration::from_secs(10), &mut implement)
14538            .await
14539            .unwrap()
14540            .unwrap();
14541        drop(implement);
14542        assert_ne!(
14543            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14544            before
14545        );
14546        assert!(git::is_clean(&winner.worktree).await.unwrap());
14547
14548        let started = tmp.path().join("pre-gate-started");
14549        let proceed = tmp.path().join("pre-gate-proceed");
14550        let checked = tmp.path().join("gate-ran");
14551        runner.state.config.verify.pre_gate = vec![format!(
14552            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14553            started.display(),
14554            proceed.display()
14555        )];
14556        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
14557        assert_eq!(
14558            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14559            0
14560        );
14561        let mut gate = Box::pin(runner.run_gate(&winner));
14562        assert!(
14563            tokio::time::timeout(Duration::from_millis(200), &mut gate)
14564                .await
14565                .is_err()
14566        );
14567        assert!(!started.exists());
14568        assert!(!checked.exists());
14569        assert!(git::is_clean(&winner.worktree).await.unwrap());
14570        assert_eq!(
14571            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14572            0
14573        );
14574        tokio::select! {
14575            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
14576            result = tokio::time::timeout(Duration::from_secs(10), async {
14577                while !started.exists() {
14578                    tokio::time::sleep(Duration::from_millis(20)).await;
14579                }
14580            }) => result.unwrap(),
14581        }
14582        assert_eq!(
14583            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14584            -1,
14585            "mechanical edits must hold the real lock"
14586        );
14587        std::fs::write(proceed, "ready").unwrap();
14588        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14589            .await
14590            .unwrap()
14591            .unwrap();
14592        drop(gate);
14593        assert!(outcomes.iter().all(CommandOutcome::ok));
14594        assert!(checked.exists());
14595        assert!(runner.state.pre_gate_commit.is_some());
14596        assert!(git::is_clean(&winner.worktree).await.unwrap());
14597        assert_eq!(
14598            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14599            0,
14600            "gate completion must release coordination"
14601        );
14602        drop(outsider);
14603        std::fs::remove_file(common.join("updater.lock")).unwrap();
14604        std::fs::create_dir(common.join("updater.lock")).unwrap();
14605        std::fs::remove_file(&started).unwrap();
14606        std::fs::remove_file(&checked).unwrap();
14607        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14608        let error = runner.run_gate(&winner).await.unwrap_err();
14609        assert!(
14610            error
14611                .to_string()
14612                .contains("acquire mandatory repository lock")
14613        );
14614        assert!(
14615            !started.exists(),
14616            "failed acquisition must prevent pre-gate"
14617        );
14618        assert!(!checked.exists(), "failed acquisition must prevent gate");
14619        assert!(git::is_clean(&winner.worktree).await.unwrap());
14620        assert_eq!(
14621            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14622            head
14623        );
14624    }
14625
14626    #[tokio::test]
14627    #[cfg(unix)]
14628    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
14629        use std::os::fd::AsRawFd as _;
14630        use std::time::Duration;
14631
14632        ask_test_home();
14633        let tmp = tempfile::tempdir().unwrap();
14634        let repo = tmp.path().join("repo");
14635        std::fs::create_dir_all(&repo).unwrap();
14636        init_repo(&repo);
14637        let common = repo.join(".git");
14638        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14639        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
14640        assert_eq!(
14641            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14642            0
14643        );
14644        let mut runner = runner_at(RunStatus::Prep);
14645        runner.state.repo = repo.clone();
14646        runner.state.config.coordination = crate::config::test_coordination();
14647        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14648        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14649        runner.state.config.graph.advise = true;
14650        runner.state.config.graph.advisors = 1;
14651        runner.roles.implementers = vec![spec("implement")];
14652        runner.roles.judges = vec![spec("judge")];
14653        let root = runner.state.worktree_root();
14654        let started = tmp.path().join("setup-started");
14655        let proceed = tmp.path().join("setup-proceed");
14656        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14657            run: Some(format!(
14658                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14659                started.display(),
14660                proceed.display()
14661            )),
14662            ..Default::default()
14663        }];
14664        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
14665        let config = git::git(&repo, &["config", "--local", "--list"])
14666            .await
14667            .unwrap();
14668        let mut prep = Box::pin(runner.prep());
14669        assert!(
14670            tokio::time::timeout(Duration::from_millis(200), &mut prep)
14671                .await
14672                .is_err()
14673        );
14674        assert!(
14675            !root.exists(),
14676            "contention must prevent even checkout creation"
14677        );
14678        assert!(!started.exists(), "contention must prevent setup");
14679        assert_eq!(
14680            git::git(&repo, &["branch", "--list"]).await.unwrap(),
14681            branches
14682        );
14683        assert_eq!(
14684            git::git(&repo, &["config", "--local", "--list"])
14685                .await
14686                .unwrap(),
14687            config
14688        );
14689        drop(lock);
14690        tokio::select! {
14691            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
14692            result = tokio::time::timeout(Duration::from_secs(10), async {
14693                while !started.exists() {
14694                    tokio::time::sleep(Duration::from_millis(20)).await;
14695                }
14696            }) => result.unwrap(),
14697        }
14698        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
14699        assert_eq!(
14700            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14701            -1
14702        );
14703        assert!(
14704            tokio::time::timeout(
14705                Duration::from_millis(100),
14706                crate::repo_lock::SeatLease::for_checkout(
14707                    &repo,
14708                    true,
14709                    crate::config::test_coordination().as_ref()
14710                )
14711            )
14712            .await
14713            .is_err(),
14714            "preparation must also exclude local editors"
14715        );
14716        std::fs::write(proceed, "ready").unwrap();
14717        tokio::time::timeout(Duration::from_secs(10), &mut prep)
14718            .await
14719            .unwrap()
14720            .unwrap();
14721        drop(prep);
14722        assert_eq!(runner.state.prep_complete, Some(true));
14723        assert!(root.join("judge-1").exists());
14724        assert!(root.join("advisor-1").exists());
14725        assert_eq!(
14726            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14727            0
14728        );
14729    }
14730
14731    #[tokio::test]
14732    #[cfg(unix)]
14733    async fn preparation_lock_failure_prevents_repository_changes() {
14734        ask_test_home();
14735        let tmp = tempfile::tempdir().unwrap();
14736        let repo = tmp.path().join("repo");
14737        std::fs::create_dir_all(&repo).unwrap();
14738        init_repo(&repo);
14739        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14740        // A directory at the required lock path makes opening the real lock
14741        // fail, rather than merely exercising an invalid registration.
14742        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
14743        let mut runner = runner_at(RunStatus::Prep);
14744        runner.state.repo = repo.clone();
14745        runner.state.config.coordination = crate::config::test_coordination();
14746        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14747        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14748        runner.roles.implementers = vec![spec("implement")];
14749        let error = runner.prep().await.unwrap_err();
14750        assert!(
14751            error
14752                .to_string()
14753                .contains("acquire mandatory repository lock")
14754        );
14755        assert!(!runner.state.worktree_root().exists());
14756        assert!(runner.state.candidates.is_empty());
14757        assert!(!repo.join(".git/config.worktree").exists());
14758        assert_eq!(
14759            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
14760            "* main"
14761        );
14762    }
14763
14764    #[tokio::test]
14765    async fn preparation_records_creation_intent_before_git_can_fail() {
14766        ask_test_home();
14767        let tmp = tempfile::tempdir().unwrap();
14768        let repo = tmp.path().join("repo");
14769        std::fs::create_dir_all(&repo).unwrap();
14770        init_repo(&repo);
14771        let mut runner = runner_at(RunStatus::Prep);
14772        runner.state.repo = repo.clone();
14773        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14774        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14775        runner.state.config.blind.commit_msg_hook = false;
14776        runner.roles.implementers = vec![spec("one")];
14777        let label = blind::assign_labels(1, runner.state.seed)[0];
14778        let branch = runner.state.branch_for(label);
14779        // A conflicting branch makes Git refuse creation before a candidate
14780        // can be registered. The durable intent must still identify it.
14781        git::git(&repo, &["branch", &branch]).await.unwrap();
14782        assert!(runner.prep().await.is_err());
14783        let saved = RunState::load(&runner.state.id).unwrap();
14784        assert!(saved.candidates.is_empty());
14785        let event = saved
14786            .events
14787            .iter()
14788            .find(|e| e.node == "prep-intent")
14789            .unwrap();
14790        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
14791        assert_eq!(intent["branch"], branch);
14792        assert_eq!(intent["start"], runner.state.base_commit);
14793        assert_eq!(
14794            intent["worktree"],
14795            runner
14796                .state
14797                .worktree_root()
14798                .join(format!("cand-{label}"))
14799                .to_string_lossy()
14800                .as_ref()
14801        );
14802        assert_eq!(intent["picks"], serde_json::json!([]));
14803    }
14804
14805    #[tokio::test]
14806    async fn invalid_pinned_sources_create_no_partial_worktrees() {
14807        ask_test_home();
14808        let tmp = tempfile::tempdir().unwrap();
14809        let repo = tmp.path().join("repo");
14810        std::fs::create_dir_all(&repo).unwrap();
14811        init_repo(&repo);
14812        let mut runner = runner_at(RunStatus::Prep);
14813        runner.state.repo = repo.clone();
14814        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14815        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14816        runner.state.sources.pick_commits.push(refs::Input {
14817            token: "missing".into(),
14818            sha: "0".repeat(40),
14819            provenance: refs::Provenance::ExplicitCli,
14820        });
14821        assert!(runner.prep().await.is_err());
14822        assert!(runner.state.candidates.is_empty());
14823        assert!(!runner.state.worktree_root().exists());
14824    }
14825
14826    #[tokio::test]
14827    async fn prose_and_legacy_inference_never_import_into_candidates() {
14828        ask_test_home();
14829        let tmp = tempfile::tempdir().unwrap();
14830        let repo = tmp.path().join("repo");
14831        std::fs::create_dir_all(&repo).unwrap();
14832        init_repo(&repo);
14833        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14834        git::git(&repo, &["checkout", "-b", "magi/history/A"])
14835            .await
14836            .unwrap();
14837        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
14838        git::git(&repo, &["add", "."]).await.unwrap();
14839        git::git(&repo, &["commit", "-m", "historical implementation"])
14840            .await
14841            .unwrap();
14842        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
14843        git::git(&repo, &["checkout", "main"]).await.unwrap();
14844        for instruction in [
14845            format!("Example: magi/history/A at {source}"),
14846            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
14847            format!("Never import magi/history/A or {source}"),
14848            format!("Conversation answer: I mentioned {source} only as background"),
14849        ] {
14850            let mut runner = runner_at(RunStatus::Prep);
14851            runner.state.repo = repo.clone();
14852            runner.state.base_commit = base.clone();
14853            runner.state.instruction = instruction.clone();
14854            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
14855            runner.state.config.blind.commit_msg_hook = false;
14856            runner.state.seeds = vec![refs::Seed {
14857                token: "magi/history/A".into(),
14858                sha: source.clone(),
14859                kind: refs::SeedKind::Unmerged,
14860                branch: true,
14861                detail: String::new(),
14862            }];
14863            runner.roles.implementers = vec![spec("one")];
14864            assert_eq!(seeded_instruction(&runner.state), instruction);
14865            // Model an old record with inferred seeds but no structured input field.
14866            let mut legacy = serde_json::to_value(&runner.state).unwrap();
14867            legacy.as_object_mut().unwrap().remove("sources");
14868            runner.state = serde_json::from_value(legacy).unwrap();
14869            assert_eq!(runner.state.sources, refs::Sources::default());
14870            runner.state.save().unwrap();
14871            runner.state = RunState::load(&runner.state.id).unwrap();
14872            runner.prep().await.unwrap();
14873            for candidate in &runner.state.candidates {
14874                assert_eq!(
14875                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
14876                    base
14877                );
14878                assert!(!candidate.worktree.join("unwanted").exists());
14879            }
14880        }
14881    }
14882
14883    #[tokio::test]
14884    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
14885        ask_test_home();
14886        let tmp = tempfile::tempdir().unwrap();
14887        let repo = tmp.path().join("repo");
14888        std::fs::create_dir_all(&repo).unwrap();
14889        init_repo(&repo);
14890        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14891        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
14892        std::fs::write(repo.join("referenced"), "required commit").unwrap();
14893        git::git(&repo, &["add", "."]).await.unwrap();
14894        git::git(&repo, &["commit", "-m", "required seed"])
14895            .await
14896            .unwrap();
14897        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
14898        git::git(&repo, &["checkout", "main"]).await.unwrap();
14899        let mut runner = runner_at(RunStatus::Prep);
14900        runner.state.repo = repo.clone();
14901        runner.state.base_commit = base.clone();
14902        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14903        runner.state.config.graph.advise = false;
14904        runner.state.config.blind.commit_msg_hook = false;
14905        runner.state.sources =
14906            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
14907                .await
14908                .unwrap();
14909        runner.roles.implementers = vec![spec("one"), spec("two")];
14910        runner.roles.judges = vec![spec("judge")];
14911        let label = blind::assign_labels(2, runner.state.seed)[0];
14912        let branch = runner.state.branch_for(label);
14913        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14914        git::worktree_add_branch(&repo, &worktree, &branch, &base)
14915            .await
14916            .unwrap();
14917        runner.state.candidates.push(Candidate {
14918            index: 0,
14919            label,
14920            agent: "one".into(),
14921            branch,
14922            worktree,
14923            summary: String::new(),
14924            stat: String::new(),
14925            files: 0,
14926            commits: 0,
14927            empty: false,
14928            failed: None,
14929            verified_noop: None,
14930            duration_ms: 0,
14931            folded: false,
14932        });
14933        runner.state.save().unwrap();
14934        // Reload the checkpoint immediately before its first cherry-pick.
14935        runner.state = RunState::load(&runner.state.id).unwrap();
14936        assert_eq!(runner.state.prep_complete, Some(false));
14937        runner.prep().await.unwrap();
14938        assert_eq!(runner.state.prep_complete, Some(true));
14939        assert_eq!(runner.state.candidates.len(), 2);
14940        for candidate in &runner.state.candidates {
14941            assert_eq!(
14942                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14943                git::tree_of(&repo, &seed).await.unwrap()
14944            );
14945            assert_eq!(
14946                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14947                "required commit"
14948            );
14949        }
14950        assert!(runner.state.worktree_root().join("judge-1").exists());
14951        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14952        runner.prep().await.unwrap();
14953        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14954
14955        runner.state.prep_complete = Some(false);
14956        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14957            run: Some(format!(
14958                "test -f '{}'",
14959                tmp.path().join("dependency-ready").display()
14960            )),
14961            ..Default::default()
14962        }];
14963        assert!(runner.prep().await.is_err());
14964        assert!(runner.state.prep_failure.is_some());
14965        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14966        runner.state = RunState::load(&runner.state.id).unwrap();
14967        assert!(
14968            runner.prep().await.is_err(),
14969            "correcting the cause alone must not auto-retry"
14970        );
14971        let failed_at = runner.state.events.last().unwrap().at;
14972        assert!(
14973            runner.retry_preparation(failed_at).is_err(),
14974            "old approvals cannot unlock the hold"
14975        );
14976        let mut task = crate::queue::Task::new(
14977            "retry".into(),
14978            "retry setup".into(),
14979            repo.clone(),
14980            crate::queue::Source::Human,
14981        );
14982        task.start(runner.state.id.clone());
14983        task.hold_machine(Some("setup failed".into()));
14984        task.release_operator("cli", None);
14985        runner
14986            .retry_preparation(task.resumption.as_ref().unwrap().at)
14987            .unwrap();
14988        runner.prep().await.unwrap();
14989        assert!(runner.state.prep_failure.is_none());
14990        assert_eq!(runner.state.prep_complete, Some(true));
14991        assert_eq!(runner.state.candidates.len(), 2);
14992        assert!(runner.state.worktree_root().join("judge-1").exists());
14993        for candidate in &runner.state.candidates {
14994            assert_eq!(
14995                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14996                git::tree_of(&repo, &seed).await.unwrap()
14997            );
14998        }
14999    }
15000
15001    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15002        let mut heads = Vec::new();
15003        for candidate in candidates {
15004            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15005        }
15006        heads
15007    }
15008
15009    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15010        ask_test_home();
15011        let tmp = tempfile::tempdir().unwrap();
15012        let repo = tmp.path().join("repo");
15013        std::fs::create_dir(&repo).unwrap();
15014        init_repo(&repo);
15015        let mut runner = runner_at(RunStatus::Reviewing);
15016        runner.state.repo = repo.clone();
15017        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15018        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15019        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15020            let branch = format!("candidate-{label}");
15021            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15022            git::worktree_add_branch(&repo, &worktree, &branch, "main")
15023                .await
15024                .unwrap();
15025            if label == 'B' {
15026                std::fs::write(worktree.join("change"), "winner").unwrap();
15027                git::rescue_commit(&worktree, "candidate change")
15028                    .await
15029                    .unwrap();
15030            }
15031            runner.state.candidates.push(Candidate {
15032                index,
15033                label,
15034                agent: "alpha".into(),
15035                branch,
15036                worktree,
15037                summary: "completed reply".into(),
15038                stat: String::new(),
15039                files: usize::from(label == 'B'),
15040                commits: usize::from(label == 'B'),
15041                empty: label == 'A',
15042                failed: None,
15043                verified_noop: None,
15044                duration_ms: 1,
15045                folded: false,
15046            });
15047        }
15048        runner.state.judge_skipped = true;
15049        runner.state.tally = Some(Tally {
15050            first_choice: BTreeMap::new(),
15051            borda: BTreeMap::new(),
15052            winner: 'B',
15053            rankings: 0,
15054            unanimous_initial: false,
15055            deliberated: false,
15056            changed_votes: 0,
15057            unanimous_final: false,
15058            tie_break: None,
15059            judges: 0,
15060            present: 0,
15061            quorum: 0,
15062            met_quorum: true,
15063            uncontested: Some("only B produced a change".into()),
15064        });
15065        runner.state.save().unwrap();
15066        (tmp, runner)
15067    }
15068
15069    #[tokio::test]
15070    #[cfg(unix)]
15071    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15072        for (solo, failed_checkpoint) in
15073            [(true, false), (false, false), (true, true), (false, true)]
15074        {
15075            let (_tmp, mut runner) = preservation_resume_fixture().await;
15076            if solo {
15077                runner.state.candidates.retain(|c| c.label == 'A');
15078            }
15079            runner.state.candidates[0].empty = false;
15080            runner.state.candidates[0].summary.clear();
15081            runner.state.tally = None;
15082            runner.state.judge_skipped = false;
15083            runner.state.status = RunStatus::Implementing;
15084            runner.state.config.disk.min_free_bytes = u64::MAX;
15085            let mut implementer = spec("alpha");
15086            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15087            runner.roles.implementers = vec![implementer];
15088            let patch = runner.state.dir().join("artifacts/cand-A.patch");
15089            if failed_checkpoint {
15090                std::fs::create_dir_all(&patch).unwrap();
15091            }
15092            runner.state.save().unwrap();
15093            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15094            runner.state = RunState::load(&runner.state.id).unwrap();
15095            assert_eq!(runner.state.status, RunStatus::Blocked);
15096            assert!(runner.state.interrupted_implementations.contains(&'A'));
15097            assert_eq!(runner.state.candidates[0].commits, 0);
15098            assert!(!runner.state.candidates[0].empty);
15099            assert!(runner.state.candidates[0].failed.is_none());
15100            if failed_checkpoint {
15101                assert!(runner.state.candidate_preservation.contains_key(&'A'));
15102                std::fs::remove_dir(patch).unwrap();
15103            }
15104            runner.state.resource_block = None;
15105            runner.state.config.disk.min_free_bytes = 0;
15106            runner.roles.implementers[0].command = vec![
15107                "sh".into(),
15108                "-c".into(),
15109                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15110            ];
15111            runner.implement().await.unwrap();
15112            assert!(runner.state.interrupted_implementations.is_empty());
15113            assert!(runner.state.candidate_preservation.is_empty());
15114            assert_eq!(runner.state.candidates[0].commits, 1);
15115            assert!(!runner.state.candidates[0].empty);
15116            assert!(
15117                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15118                    .unwrap()
15119                    .contains("retried empty interrupted seat")
15120            );
15121            if !solo {
15122                assert_eq!(runner.state.candidates[1].commits, 1);
15123            }
15124        }
15125    }
15126
15127    #[tokio::test]
15128    #[cfg(unix)]
15129    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15130        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15131            let (_tmp, mut runner) = preservation_resume_fixture().await;
15132            runner.state.candidates.retain(|c| c.label == 'A');
15133            runner.state.candidates[0].empty = false;
15134            runner.state.candidates[0].summary.clear();
15135            runner.state.tally = None;
15136            runner.state.judge_skipped = false;
15137            runner.state.status = RunStatus::Implementing;
15138            // Force the real post-spawn watchdog, independently of available
15139            // disk space. Dirty/committed work predates its first measurement.
15140            runner.state.config.disk.min_free_bytes = u64::MAX;
15141            let wt = runner.state.candidates[0].worktree.clone();
15142            std::fs::write(
15143                wt.join("interrupted-source"),
15144                "retain implementation work\n",
15145            )
15146            .unwrap();
15147            if committed {
15148                git::rescue_commit(&wt, "implementation checkpoint")
15149                    .await
15150                    .unwrap();
15151            }
15152            let lock = git::git(
15153                &wt,
15154                &[
15155                    "rev-parse",
15156                    "--path-format=absolute",
15157                    "--git-path",
15158                    "index.lock",
15159                ],
15160            )
15161            .await
15162            .unwrap();
15163            if block_rescue {
15164                std::fs::write(&lock, "fixture-owned lock").unwrap();
15165            }
15166            let mut implementer = spec("alpha");
15167            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15168            runner.roles.implementers = vec![implementer];
15169            runner.state.save().unwrap();
15170            let result = runner.implement().await;
15171            assert_eq!(result.is_err(), block_rescue);
15172            assert_eq!(
15173                runner.state.resource_block.as_ref().unwrap().kind,
15174                "disk pressure"
15175            );
15176            assert!(
15177                runner
15178                    .state
15179                    .events
15180                    .iter()
15181                    .any(|e| e.message.contains("agent execution interrupted"))
15182            );
15183            assert!(
15184                runner
15185                    .state
15186                    .dir()
15187                    .join("artifacts/candidate-A-recovery/staged.patch")
15188                    .exists()
15189            );
15190            if block_rescue {
15191                let saved = RunState::load(&runner.state.id).unwrap();
15192                assert!(saved.candidate_preservation.contains_key(&'A'));
15193                assert!(!saved.candidates[0].empty);
15194                assert_eq!(
15195                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15196                    "retain implementation work\n"
15197                );
15198                std::fs::remove_file(lock).unwrap();
15199                runner.state = saved;
15200            } else {
15201                assert_eq!(runner.state.status, RunStatus::Blocked);
15202                assert_eq!(runner.state.candidates[0].commits, 1);
15203            }
15204            // Recovery must save the candidate before deciding whether another
15205            // implementation is needed. No implementer is available on retry.
15206            runner.roles.implementers.clear();
15207            runner.state.config.disk.min_free_bytes = 0;
15208            runner.state.resource_block = None;
15209            runner.implement().await.unwrap();
15210            assert!(runner.state.candidate_preservation.is_empty());
15211            assert_eq!(runner.state.candidates[0].commits, 1);
15212            assert!(!runner.state.candidates[0].empty);
15213            assert!(git::is_clean(&wt).await.unwrap());
15214            assert!(
15215                crate::run::read_artifact(&runner.state, "cand-A.patch")
15216                    .unwrap()
15217                    .contains("retain implementation work")
15218            );
15219        }
15220    }
15221
15222    #[tokio::test]
15223    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15224        let (_tmp, mut runner) = preservation_resume_fixture().await;
15225        // Schema 15 already confirmed the unfolded no-change result.
15226        runner.implement().await.unwrap();
15227        assert!(runner.state.candidate_preservation.is_empty());
15228        assert!(
15229            !runner
15230                .state
15231                .dir()
15232                .join("artifacts/candidate-A-recovery")
15233                .exists()
15234        );
15235        runner.fold_losers().await.unwrap();
15236        assert!(!runner.state.candidates[0].worktree.exists());
15237        for schema in [14, crate::run::SCHEMA] {
15238            runner.state.schema = schema;
15239            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15240            runner.state.save().unwrap();
15241            runner.state = RunState::load(&runner.state.id).unwrap();
15242            runner.implement().await.unwrap();
15243            assert!(runner.state.candidate_preservation.is_empty());
15244            assert!(runner.state.candidates[0].folded);
15245            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15246        }
15247    }
15248
15249    #[tokio::test]
15250    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15251        let (_tmp, mut runner) = preservation_resume_fixture().await;
15252        runner.state.candidates.retain(|c| c.label == 'A');
15253        runner.state.tally = None;
15254        runner.state.judge_skipped = false;
15255        let candidate = &mut runner.state.candidates[0];
15256        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15257        candidate.failed = Some("deployment validation unavailable".into());
15258        std::fs::write(
15259            candidate.worktree.join("repair"),
15260            "future launcher policy\n",
15261        )
15262        .unwrap();
15263        runner
15264            .state
15265            .candidate_preservation
15266            .insert('A', "awaiting persistence".into());
15267        runner.persist_candidates().await.unwrap();
15268        let candidate = &runner.state.candidates[0];
15269        assert!(!candidate.empty);
15270        assert_eq!(candidate.commits, 1);
15271        assert_eq!(candidate.files, 1);
15272        assert!(candidate.failed.is_none());
15273        assert!(candidate.summary.contains("EPERM"));
15274        assert!(
15275            crate::run::read_artifact(&runner.state, "cand-A.patch")
15276                .unwrap()
15277                .contains("future launcher policy")
15278        );
15279        assert!(runner.state.candidate_preservation.is_empty());
15280    }
15281
15282    #[tokio::test]
15283    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
15284        let (_tmp, mut runner) = preservation_resume_fixture().await;
15285        let wt = runner.state.candidates[0].worktree.clone();
15286        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
15287        runner.state.schema = 14;
15288        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15289        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15290        let tally = serde_json::to_value(&runner.state.tally).unwrap();
15291        for _ in 0..2 {
15292            assert!(runner.implement().await.is_err());
15293            assert_eq!(runner.state.status, RunStatus::Blocked);
15294            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15295            assert!(
15296                runner.state.candidates[0].empty,
15297                "old verdict must remain unusable"
15298            );
15299            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
15300            assert!(runner.state.judge_skipped);
15301            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
15302            assert!(
15303                crate::run::read_artifact(&runner.state, "cand-A.patch")
15304                    .unwrap()
15305                    .contains("owned edits")
15306            );
15307            assert!(wt.exists());
15308            assert_eq!(
15309                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
15310                    .await
15311                    .unwrap(),
15312                1
15313            );
15314            runner.state = RunState::load(&runner.state.id).unwrap();
15315        }
15316    }
15317
15318    #[tokio::test]
15319    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
15320        let (_tmp, mut runner) = preservation_resume_fixture().await;
15321        let wt = runner.state.candidates[0].worktree.clone();
15322        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
15323        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
15324        let index_lock = PathBuf::from(
15325            git::git(
15326                &wt,
15327                &[
15328                    "rev-parse",
15329                    "--path-format=absolute",
15330                    "--git-path",
15331                    "index.lock",
15332                ],
15333            )
15334            .await
15335            .unwrap(),
15336        );
15337        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15338        runner.state.schema = 14;
15339        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15340        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15341        assert!(runner.implement().await.is_err());
15342        assert!(runner.after_implement().is_err());
15343        assert!(runner.fold_losers().await.is_err());
15344        assert_eq!(
15345            std::fs::read_to_string(wt.join("recovered")).unwrap(),
15346            "failed seat edits"
15347        );
15348        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15349        runner.state = RunState::load(&runner.state.id).unwrap();
15350        std::fs::remove_file(index_lock).unwrap();
15351        // Recovery finds changes that the old uncontested result never judged.
15352        assert!(runner.implement().await.is_err());
15353        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15354        assert!(runner.state.candidates[0].empty);
15355        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15356        assert!(
15357            crate::run::read_artifact(&runner.state, "cand-A.patch")
15358                .unwrap()
15359                .contains("failed seat edits")
15360        );
15361        assert!(wt.exists());
15362    }
15363
15364    #[tokio::test]
15365    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
15366        let (_tmp, mut runner) = preservation_resume_fixture().await;
15367        let repo = runner.state.repo.clone();
15368        let common = git::git(
15369            &repo,
15370            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
15371        )
15372        .await
15373        .unwrap();
15374        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
15375        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
15376        assert!(runner.fold_losers().await.is_err());
15377        assert!(!runner.state.candidates[0].worktree.exists());
15378        assert!(!runner.state.candidates[0].folded);
15379        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
15380        runner.state = RunState::load(&runner.state.id).unwrap();
15381        std::fs::remove_file(branch_lock).unwrap();
15382        let judge = runner.state.worktree_root().join("judge-1");
15383        git::worktree_add_detached(&repo, &judge, "main")
15384            .await
15385            .unwrap();
15386        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
15387        runner.roles.judges = vec![spec("judge")];
15388        assert!(runner.fold_losers().await.is_err());
15389        runner.state = RunState::load(&runner.state.id).unwrap();
15390        assert!(
15391            runner.state.candidates[0].folded,
15392            "candidate progress saved before judge failure"
15393        );
15394        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
15395        assert!(judge.join("uncommitted").exists());
15396        std::fs::remove_file(judge.join("uncommitted")).unwrap();
15397        runner.fold_losers().await.unwrap();
15398        assert!(!judge.exists());
15399        runner.fold_losers().await.unwrap();
15400    }
15401
15402    #[tokio::test]
15403    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
15404        let (_tmp, mut runner) = preservation_resume_fixture().await;
15405        runner.state.candidates.retain(|c| c.label == 'B');
15406        runner.state.tally = None;
15407        runner.state.judge_skipped = false;
15408        runner.state.prep_complete = Some(true);
15409        runner.state.config.graph.advise = false;
15410        runner.state.config.graph.selected_implementer = Some("alpha".into());
15411        let wt = runner.state.candidates[0].worktree.clone();
15412        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
15413        runner
15414            .state
15415            .candidate_preservation
15416            .insert('B', "awaiting persistence".into());
15417        let index_lock = git::git(
15418            &wt,
15419            &[
15420                "rev-parse",
15421                "--path-format=absolute",
15422                "--git-path",
15423                "index.lock",
15424            ],
15425        )
15426        .await
15427        .unwrap();
15428        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15429        runner.state.save().unwrap();
15430        assert!(runner.execute_graph().await.is_err());
15431        assert_eq!(runner.state.status, RunStatus::Blocked);
15432        assert!(runner.state.candidate_preservation.contains_key(&'B'));
15433        std::fs::remove_file(index_lock).unwrap();
15434        // Park at the next graph boundary: this tests the real selected-seat
15435        // hold guard without dispatching reviewers or publishing anything.
15436        for _ in 0..2 {
15437            runner.state = RunState::load(&runner.state.id).unwrap();
15438            runner.pause.park();
15439            runner.execute_graph().await.unwrap();
15440            assert_eq!(runner.state.status, RunStatus::Implementing);
15441            assert!(runner.state.candidate_preservation.is_empty());
15442            assert_eq!(runner.state.candidates[0].commits, 2);
15443            assert!(runner.state.tally.is_none());
15444        }
15445        runner.implement().await.unwrap();
15446        assert_eq!(runner.state.status, RunStatus::Judging);
15447        // A genuine selected-implementer hold must still require release.
15448        runner.state.status = RunStatus::Blocked;
15449        runner.state.candidates[0].failed = Some("implementation failed".into());
15450        runner.state.save().unwrap();
15451        let error = runner.execute_graph().await.unwrap_err();
15452        assert!(
15453            error
15454                .to_string()
15455                .contains("selected implementation is held")
15456        );
15457    }
15458
15459    #[tokio::test]
15460    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
15461        let (_tmp, mut runner) = preservation_resume_fixture().await;
15462        let repo = runner.state.repo.clone();
15463        let candidate = runner.state.candidates[0].clone();
15464        // Remove only the disposable fixture directory, leaving Git's linked
15465        // worktree registration and branch intact, as in a vanished checkout.
15466        let registered = git::worktree_holding(&repo, &candidate.branch)
15467            .await
15468            .unwrap()
15469            .unwrap();
15470        assert_eq!(
15471            registered.canonicalize().unwrap(),
15472            candidate.worktree.canonicalize().unwrap()
15473        );
15474        std::fs::remove_dir_all(&candidate.worktree).unwrap();
15475        assert_eq!(
15476            git::worktree_holding(&repo, &candidate.branch)
15477                .await
15478                .unwrap(),
15479            Some(registered)
15480        );
15481        assert!(
15482            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
15483            "the stale registration initially keeps the branch checked out"
15484        );
15485        runner.fold_losers().await.unwrap();
15486        assert!(runner.state.candidates[0].folded);
15487        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15488        assert_eq!(
15489            git::worktree_holding(&repo, &candidate.branch)
15490                .await
15491                .unwrap(),
15492            None
15493        );
15494        runner.state = RunState::load(&runner.state.id).unwrap();
15495        runner.fold_losers().await.unwrap();
15496    }
15497
15498    #[tokio::test]
15499    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
15500        let (_tmp, mut runner) = preservation_resume_fixture().await;
15501        let repo = runner.state.repo.clone();
15502        let candidate = runner.state.candidates[0].clone();
15503        let winner = runner.state.candidates[1].clone();
15504        let registered = git::worktree_holding(&repo, &candidate.branch)
15505            .await
15506            .unwrap()
15507            .unwrap();
15508        assert_eq!(
15509            registered.canonicalize().unwrap(),
15510            candidate.worktree.canonicalize().unwrap()
15511        );
15512        std::fs::remove_dir_all(&candidate.worktree).unwrap();
15513        assert_eq!(
15514            git::worktree_holding(&repo, &candidate.branch)
15515                .await
15516                .unwrap(),
15517            Some(registered)
15518        );
15519        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
15520
15521        for _ in 0..2 {
15522            fold_run(&mut runner.state, false, &crate::run::home())
15523                .await
15524                .unwrap();
15525            runner.state = RunState::load(&runner.state.id).unwrap();
15526            assert!(runner.state.candidates[0].folded);
15527            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15528            assert_eq!(
15529                git::worktree_holding(&repo, &candidate.branch)
15530                    .await
15531                    .unwrap(),
15532                None
15533            );
15534            assert!(!runner.state.candidates[1].folded);
15535            assert!(winner.worktree.exists());
15536            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
15537        }
15538    }
15539
15540    #[tokio::test]
15541    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
15542        ask_test_home();
15543        let tmp = tempfile::tempdir().unwrap();
15544        let repo = tmp.path().join("repo");
15545        std::fs::create_dir(&repo).unwrap();
15546        init_repo(&repo);
15547        let mut runner = runner_at(RunStatus::Implementing);
15548        runner.state.repo = repo.clone();
15549        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15550        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15551        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15552            let branch = format!("candidate-{label}");
15553            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
15554            git::worktree_add_branch(&repo, &wt, &branch, "main")
15555                .await
15556                .unwrap();
15557            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
15558            git::git(&wt, &["add", "README.md"]).await.unwrap();
15559            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
15560            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
15561            runner.state.candidates.push(Candidate {
15562                index,
15563                label,
15564                agent: "alpha".to_owned(),
15565                branch,
15566                worktree: wt,
15567                summary: "completed implementation".to_owned(),
15568                stat: String::new(),
15569                files: 0,
15570                commits: 0,
15571                empty: false,
15572                failed: None,
15573                verified_noop: None,
15574                duration_ms: 42,
15575                folded: false,
15576            });
15577            runner
15578                .state
15579                .candidate_preservation
15580                .insert(label, "awaiting persistence".to_owned());
15581        }
15582        let wt = runner.state.candidates[0].worktree.clone();
15583        let index_lock = git::git(
15584            &wt,
15585            &[
15586                "rev-parse",
15587                "--path-format=absolute",
15588                "--git-path",
15589                "index.lock",
15590            ],
15591        )
15592        .await
15593        .unwrap();
15594        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15595        runner.state.save().unwrap();
15596        assert!(runner.persist_candidates().await.is_err());
15597        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15598        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
15599        assert_eq!(
15600            runner.state.candidates[1].commits, 1,
15601            "other results are still collected"
15602        );
15603        assert!(
15604            !runner.state.candidates[0].empty,
15605            "failed rescue is not no change"
15606        );
15607        assert_eq!(
15608            runner.state.candidates[0].summary,
15609            "completed implementation"
15610        );
15611        assert!(runner.after_implement().is_err());
15612        let home = tmp.path().join("fold-home");
15613        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
15614        assert!(wt.join("new-file").exists());
15615        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
15616        assert!(recovery.join("staged.patch").exists());
15617        assert!(recovery.join("unstaged.patch").exists());
15618        assert_eq!(
15619            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
15620            "untracked edits\n"
15621        );
15622        runner.state.begin_pass();
15623        runner.state.end_pass_parked(ParkCause::Upgrade {
15624            from: "1.0".into(),
15625            to: Some("1.1".into()),
15626        });
15627        runner.state.save().unwrap();
15628        runner.state = RunState::load(&runner.state.id).unwrap();
15629        assert!(matches!(
15630            runner.state.passes.last().unwrap().end,
15631            Some(crate::run::PassEnd::Parked {
15632                cause: ParkCause::Upgrade { .. }
15633            })
15634        ));
15635        std::fs::remove_file(index_lock).unwrap();
15636        // No implementer is configured: a persistence retry cannot dispatch a seat.
15637        runner.implement().await.unwrap();
15638        assert!(runner.state.candidate_preservation.is_empty());
15639        assert!(
15640            runner
15641                .state
15642                .candidates
15643                .iter()
15644                .all(|c| c.commits == 1 && !c.empty)
15645        );
15646        assert!(git::is_clean(&wt).await.unwrap());
15647        assert!(
15648            crate::run::read_artifact(&runner.state, "cand-A.patch")
15649                .unwrap()
15650                .contains("later tracked edits")
15651        );
15652    }
15653
15654    #[tokio::test]
15655    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
15656        ask_test_home();
15657        let tmp = tempfile::tempdir().unwrap();
15658        let repo = tmp.path().join("repo");
15659        std::fs::create_dir(&repo).unwrap();
15660        init_repo(&repo);
15661        let wt = tmp.path().join("candidate");
15662        git::worktree_add_branch(&repo, &wt, "candidate", "main")
15663            .await
15664            .unwrap();
15665        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15666        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
15667        assert!(wt.join("uncommitted").exists());
15668        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15669        std::fs::remove_file(wt.join("uncommitted")).unwrap();
15670        let mut runner = runner_at(RunStatus::Implementing);
15671        runner.state.repo = repo.clone();
15672        runner.state.base_commit = "missing-base".to_owned();
15673        runner.state.candidates.push(Candidate {
15674            index: 0,
15675            label: 'A',
15676            agent: "alpha".into(),
15677            branch: "candidate".into(),
15678            worktree: wt.clone(),
15679            summary: "completed".into(),
15680            stat: String::new(),
15681            files: 0,
15682            commits: 0,
15683            empty: false,
15684            failed: None,
15685            verified_noop: None,
15686            duration_ms: 1,
15687            folded: false,
15688        });
15689        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15690        assert!(
15691            fold_run(&mut runner.state, true, &tmp.path().join("home"))
15692                .await
15693                .is_err()
15694        );
15695        assert!(!runner.state.candidates[0].folded);
15696        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15697        std::fs::remove_file(wt.join("uncommitted")).unwrap();
15698        runner
15699            .state
15700            .candidate_preservation
15701            .insert('A', "pending".into());
15702        assert!(runner.persist_candidates().await.is_err());
15703        assert!(!runner.state.candidates[0].empty);
15704        assert!(!runner.state.candidates[0].folded);
15705        assert!(wt.exists());
15706        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15707        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
15708        std::fs::create_dir_all(&patch_path).unwrap();
15709        assert!(
15710            runner.persist_candidates().await.is_err(),
15711            "artifact failure stays pending"
15712        );
15713        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15714        assert!(!runner.state.candidates[0].empty);
15715        std::fs::remove_dir(patch_path).unwrap();
15716        runner.persist_candidates().await.unwrap();
15717        assert!(
15718            runner.state.candidates[0].empty,
15719            "only successful inspection confirms no change"
15720        );
15721    }
15722
15723    fn init_repo(dir: &Path) {
15724        let run = |args: &[&str]| {
15725            let out = std::process::Command::new("git")
15726                .args(args)
15727                .current_dir(dir)
15728                .quiet()
15729                .output()
15730                .expect("spawn git");
15731            assert!(
15732                out.status.success(),
15733                "git {args:?} failed: {}",
15734                String::from_utf8_lossy(&out.stderr)
15735            );
15736        };
15737        run(&["init", "-b", "main"]);
15738        run(&["config", "user.name", "magi test"]);
15739        run(&["config", "user.email", "magi@example.com"]);
15740        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
15741        run(&["add", "-A"]);
15742        run(&["commit", "-m", "init"]);
15743    }
15744
15745    // `settle_questions` is what closes the ghost the phone showed: a run's
15746    // seat asked something, the run then ended, and nothing was left to
15747    // abandon the question it left `open`. `HOME` is a process-wide
15748    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
15749    // first time it runs in the binary — every test below still reaches the
15750    // same directory whichever call won, and each gets its own run id from
15751    // `RunState::new`, so they never collide there.
15752    fn ask_test_home() {
15753        crate::run::pin_test_home();
15754    }
15755
15756    #[test]
15757    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
15758        ask_test_home();
15759        let mut runner = runner_at(RunStatus::Prep);
15760        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
15761        // A blocked temporary output path makes the terminal save fail while
15762        // preserving the already published reservation.
15763        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
15764        let error = guard
15765            .finish(
15766                &mut runner.state,
15767                Err(
15768                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
15769                        .into(),
15770                ),
15771            )
15772            .unwrap_err();
15773        assert_eq!(
15774            error.downcast_ref::<std::io::Error>().unwrap().kind(),
15775            std::io::ErrorKind::PermissionDenied
15776        );
15777        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
15778        let message = format!("{error:#}");
15779        assert!(message.contains("original refusal"));
15780        assert!(!message.contains("could not save terminal"));
15781        assert_eq!(
15782            error.chain().count(),
15783            2,
15784            "only startup attribution wraps the original error"
15785        );
15786        let saved = RunState::load(&runner.state.id).unwrap();
15787        assert_eq!(saved.status, RunStatus::Blocked);
15788        assert!(!saved.startup.unwrap().complete);
15789    }
15790
15791    #[test]
15792    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
15793        ask_test_home();
15794        let mut runner = runner_at(RunStatus::Prep);
15795        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
15796        let id = runner.state.id.clone();
15797        drop(guard);
15798        let saved = RunState::load(&id).unwrap();
15799        assert_eq!(saved.status, RunStatus::Blocked);
15800        assert!(!saved.startup.as_ref().unwrap().complete);
15801        assert!(
15802            saved
15803                .startup
15804                .unwrap()
15805                .failure
15806                .unwrap()
15807                .contains("interrupted")
15808        );
15809        assert!(Runner::resume(&id).is_err());
15810    }
15811
15812    #[test]
15813    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
15814        ask_test_home();
15815        let mut runner = runner_at(RunStatus::Prep);
15816        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
15817        let error = guard
15818            .finish(
15819                &mut runner.state,
15820                Err(
15821                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
15822                        .into(),
15823                ),
15824            )
15825            .unwrap_err();
15826        assert_eq!(
15827            error.downcast_ref::<std::io::Error>().unwrap().kind(),
15828            std::io::ErrorKind::PermissionDenied
15829        );
15830        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
15831        let mut task = crate::queue::Task::new(
15832            "review".into(),
15833            "criteria".into(),
15834            runner.state.repo.clone(),
15835            crate::queue::Source::Human,
15836        );
15837        task.record_startup_failure(failure.clone());
15838        task.record_startup_failure(failure.clone());
15839        assert_eq!(task.startup_failures.len(), 1);
15840        assert!(task.runs.is_empty());
15841        assert_eq!(task.attempts, 0);
15842    }
15843
15844    #[cfg(unix)]
15845    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
15846        ask_test_home();
15847        let tmp = tempfile::tempdir().unwrap();
15848        let repo = tmp.path().join("repo");
15849        std::fs::create_dir(&repo).unwrap();
15850        init_repo(&repo);
15851        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15852        let origin = tmp.path().join("origin.git");
15853        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
15854            .await
15855            .unwrap();
15856        git::git(
15857            &repo,
15858            &["remote", "add", "origin", origin.to_str().unwrap()],
15859        )
15860        .await
15861        .unwrap();
15862        git::git(&repo, &["checkout", "-b", "fix/retained"])
15863            .await
15864            .unwrap();
15865        std::fs::write(repo.join("retained"), "retained change").unwrap();
15866        git::git(&repo, &["add", "."]).await.unwrap();
15867        git::git(&repo, &["commit", "-m", "retained implementation"])
15868            .await
15869            .unwrap();
15870        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15871        git::git(&repo, &["checkout", "main"]).await.unwrap();
15872        std::fs::write(repo.join("base-change"), "new base").unwrap();
15873        git::git(&repo, &["add", "."]).await.unwrap();
15874        git::git(&repo, &["commit", "-m", "base moved"])
15875            .await
15876            .unwrap();
15877        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15878        git::git(&repo, &["checkout", "fix/retained"])
15879            .await
15880            .unwrap();
15881        let mut runner = runner_at(RunStatus::Voting);
15882        runner.state.repo = repo.clone();
15883        runner.state.base_commit = base;
15884        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15885        runner.state.config.disk.min_free_bytes = 0;
15886        runner.state.config.graph.review_rounds = 1;
15887        runner.state.config.verify.e2e = vec!["true".into()];
15888        runner.state.config.verify.gate = vec!["true".into()];
15889        runner.state.candidates = vec![Candidate {
15890            index: 0,
15891            label: 'A',
15892            agent: "fixture".into(),
15893            branch: "fix/retained".into(),
15894            worktree: repo.clone(),
15895            summary: String::new(),
15896            stat: String::new(),
15897            files: 1,
15898            commits: 1,
15899            empty: false,
15900            failed: None,
15901            verified_noop: None,
15902            duration_ms: 0,
15903            folded: false,
15904        }];
15905        runner.state.tally = Some(Tally {
15906            first_choice: BTreeMap::new(),
15907            borda: BTreeMap::new(),
15908            winner: 'A',
15909            rankings: 0,
15910            unanimous_initial: true,
15911            deliberated: false,
15912            changed_votes: 0,
15913            unanimous_final: true,
15914            tie_break: None,
15915            judges: 0,
15916            present: 0,
15917            quorum: 0,
15918            met_quorum: true,
15919            uncontested: Some("review-only".into()),
15920        });
15921        let mut task = crate::queue::Task::new(
15922            "fixture".into(),
15923            "fixture".into(),
15924            repo,
15925            crate::queue::Source::Human,
15926        );
15927        task.link_run(&runner.state.id);
15928        crate::queue::Queue::open().put(&mut task).unwrap();
15929        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
15930        runner.state.base_sync = Some(BaseSync {
15931            tip: String::new(),
15932            behind: 1,
15933            attempts: BASE_SYNC_ROUNDS,
15934            conflict: None,
15935            already_in: None,
15936        });
15937        runner.sync_to_base().await.unwrap();
15938        assert_eq!(runner.state.status, RunStatus::Blocked);
15939        (tmp, runner, head, task.id)
15940    }
15941
15942    #[cfg(unix)]
15943    #[tokio::test]
15944    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
15945        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
15946        if std::env::var_os(CHILD).is_none() {
15947            let home = tempfile::tempdir().unwrap();
15948            let output = std::process::Command::new(std::env::current_exe().unwrap())
15949                .args([
15950                    "--exact",
15951                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
15952                    "--nocapture",
15953                ])
15954                .env(CHILD, "1")
15955                .env("MAGI_HOME", home.path())
15956                .env("MAGI_WORKER_HOME", home.path())
15957                .env("MAGI_WORKER_ID", "abcdef")
15958                .output()
15959                .unwrap();
15960            assert!(
15961                output.status.success()
15962                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15963                "{}\n{}",
15964                String::from_utf8_lossy(&output.stdout),
15965                String::from_utf8_lossy(&output.stderr)
15966            );
15967            return;
15968        }
15969        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
15970        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15971        let home = crate::run::home();
15972        let queue = crate::queue::Queue::open();
15973        let mut owner = queue.get(&task).unwrap();
15974        owner.release_operator("test-release", None);
15975        owner.start(runner.state.id.clone());
15976        queue.put(&mut owner).unwrap();
15977        runner.state.driver_exited = true;
15978        runner.state.task_instruction = Some(owner.instruction.clone());
15979        runner.state.save().unwrap();
15980        let _claim = queue.claim(&task).unwrap();
15981        let reserve = |id: &str| {
15982            let directory = home.join("workers/jobs").join(id);
15983            std::fs::create_dir_all(&directory).unwrap();
15984            let job = crate::worker::Job {
15985                id: id.into(),
15986                sequence: 1,
15987                kind: "task".into(),
15988                key: format!("task:{task}"),
15989                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
15990                generation: "fixture-generation".into(),
15991                version: None,
15992                status: crate::worker::Status::Running,
15993                result: None,
15994                pid: Some(std::process::id()),
15995                created_at: 0,
15996                finished_at: None,
15997            };
15998            std::fs::write(
15999                directory.join("job.json"),
16000                serde_json::to_vec(&job).unwrap(),
16001            )
16002            .unwrap();
16003            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16004        };
16005        let _own = reserve("abcdef");
16006        let other = reserve("fedcba");
16007        assert!(
16008            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16009                .await
16010                .is_err()
16011        );
16012        assert!(
16013            RunState::load(&runner.state.id)
16014                .unwrap()
16015                .base_sync_recovery
16016                .authorizations
16017                .is_empty()
16018        );
16019        drop(other);
16020        assert!(crate::daemon::is_working_on(
16021            &home,
16022            &runner.state.id,
16023            Timestamp::now()
16024        ));
16025        assert!(
16026            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16027        );
16028        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16029            .await
16030            .unwrap();
16031        assert!(
16032            RunState::load(&runner.state.id)
16033                .unwrap()
16034                .base_recovery_pending()
16035        );
16036    }
16037
16038    #[cfg(unix)]
16039    #[tokio::test]
16040    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16041        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16042        let home = crate::run::home();
16043        runner.state.driver_pid = Some(std::process::id());
16044        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16045        runner.state.driver_exited = false;
16046        runner.state.begin_pass();
16047        runner.state.save().unwrap();
16048        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16049        driver.record(&runner.state).unwrap();
16050        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16051        // Preserve the durable claim as if the process died before Drop saved.
16052        drop(driver);
16053        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16054        assert_eq!(
16055            runner.state.liveness_under(&home, false),
16056            crate::run::Liveness::Dead
16057        );
16058        let queue = crate::queue::Queue::open();
16059        let _claim = queue.claim(&task).unwrap();
16060        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16061            .await
16062            .unwrap();
16063        let disk = RunState::load(&runner.state.id).unwrap();
16064        assert!(!disk.driver_exited);
16065        assert!(disk.base_recovery_pending());
16066    }
16067
16068    #[cfg(unix)]
16069    #[tokio::test]
16070    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16071        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16072        if std::env::var_os(CHILD).is_none() {
16073            use std::os::unix::fs::PermissionsExt;
16074            let tmp = tempfile::tempdir().unwrap();
16075            let gh = tmp.path().join("gh");
16076            std::fs::write(&gh, r#"#!/bin/sh
16077case "$1 $2" in
16078  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16079  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16080  *) exit 2 ;;
16081esac
16082"#).unwrap();
16083            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16084            let mut paths = vec![tmp.path().to_path_buf()];
16085            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16086            let output = std::process::Command::new(std::env::current_exe().unwrap())
16087                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16088                .env(CHILD, "1")
16089                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16090                .env("PATH", std::env::join_paths(paths).unwrap())
16091                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16092                .output().unwrap();
16093            assert!(
16094                output.status.success()
16095                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16096                "{}\n{}",
16097                String::from_utf8_lossy(&output.stdout),
16098                String::from_utf8_lossy(&output.stderr)
16099            );
16100            return;
16101        }
16102        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16103        let repo = runner.state.repo.clone();
16104        git::git(&repo, &["checkout", "main"]).await.unwrap();
16105        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16106        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16107        git::git(&repo, &["checkout", "fix/retained"])
16108            .await
16109            .unwrap();
16110        runner.state.pr = Some(crate::run::PrRecord {
16111            url: "https://example.test/project/pull/1".into(),
16112            number: 1,
16113            state: "open".into(),
16114            checks: "unknown".into(),
16115            round: 0,
16116            rounds: 1,
16117            red_at_merge: Vec::new(),
16118        });
16119        runner.sync_to_base().await.unwrap();
16120        assert_eq!(runner.state.status, RunStatus::Blocked);
16121        assert_eq!(
16122            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16123            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16124        );
16125        assert!(runner.state.base_sync_closure_pending());
16126        assert!(crate::handover::can_resume(&runner.state));
16127        let queue = crate::queue::Queue::open();
16128        assert!(!crate::base_sync_recovery::held_task(
16129            &queue,
16130            &queue.get(&task).unwrap()
16131        ));
16132        assert!(runner.state.authorize_base_sync(&task).is_err());
16133        // Legacy closure stops have no typed stop but must retain the checkpoint.
16134        runner.state.base_sync_recovery.stop = None;
16135        runner.state.status = RunStatus::Voting;
16136        runner.state.gate_ran = true;
16137        runner.state.config.graph.review_rounds = 0;
16138        runner.state.save().unwrap();
16139        assert!(crate::handover::can_resume(&runner.state));
16140        runner.state = RunState::load(&runner.state.id).unwrap();
16141        runner.execute_graph().await.unwrap();
16142        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16143        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16144        assert_eq!(
16145            runner.state.base_sync.as_ref().unwrap().attempts,
16146            BASE_SYNC_ROUNDS
16147        );
16148        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16149        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16150        assert!(runner.state.reviews.is_empty());
16151        assert!(runner.state.gate.is_empty());
16152    }
16153
16154    #[cfg(unix)]
16155    #[tokio::test]
16156    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16157        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16158        runner.state.driver_exited = true;
16159        runner.state.save().unwrap();
16160        let home = crate::run::home();
16161        let queue = crate::queue::Queue::open();
16162        let _claim = queue.claim(&task).unwrap();
16163        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16164        assert!(
16165            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16166                .await
16167                .is_err()
16168        );
16169        drop(lease);
16170        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16171        assert!(
16172            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16173                .await
16174                .is_err()
16175        );
16176        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16177        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16178            .await
16179            .unwrap();
16180        let disk = RunState::load(&runner.state.id).unwrap();
16181        assert!(disk.base_recovery_pending());
16182        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16183        assert!(
16184            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16185                .await
16186                .is_err()
16187        );
16188        assert_eq!(
16189            RunState::load(&runner.state.id)
16190                .unwrap()
16191                .base_sync_recovery
16192                .authorizations
16193                .len(),
16194            1
16195        );
16196        let mut owner = queue.get(&task).unwrap();
16197        owner.runs.push("20260101-000000-ffff".into());
16198        queue.put(&mut owner).unwrap();
16199        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16200            .await
16201            .unwrap_err();
16202        assert!(error.to_string().contains("run task ownership changed"));
16203    }
16204
16205    #[cfg(unix)]
16206    #[tokio::test]
16207    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16208        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16209        let mut legacy = serde_json::to_value(&runner.state).unwrap();
16210        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16211        legacy["status"] = serde_json::json!("voting");
16212        runner.state = serde_json::from_value(legacy).unwrap();
16213        runner.state.save().unwrap();
16214        for _ in 0..3 {
16215            runner.execute_graph().await.unwrap();
16216            assert_eq!(runner.state.status, RunStatus::Blocked);
16217            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16218            assert!(!crate::handover::can_resume(&runner.state));
16219            assert_eq!(
16220                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16221                head
16222            );
16223            assert!(runner.state.reviews.is_empty());
16224            assert!(!runner.state.gate_ran);
16225        }
16226        let notice = crate::notices::run_ended(&runner.state).unwrap();
16227        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16228        runner.state.begin_pass();
16229        runner.state.status = RunStatus::Voting;
16230        runner.deliberate().await.unwrap();
16231        runner.vote().await.unwrap();
16232        runner.finish_after_tally().await.unwrap();
16233        assert_eq!(runner.state.status, RunStatus::Blocked);
16234        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16235        assert_eq!(
16236            crate::notices::run_ended(&runner.state).unwrap().since,
16237            notice.since
16238        );
16239    }
16240
16241    #[cfg(unix)]
16242    #[tokio::test]
16243    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16244        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16245        for detail in [
16246            "fix/retained conflicts with origin/main and did not rebase: conflict",
16247            "rebased fix/retained locally but origin refused the push",
16248            "unknown failure",
16249        ] {
16250            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16251            runner.state.record_base_sync_stop(&head);
16252            assert!(runner.state.authorize_base_sync(&task).is_err());
16253            runner.state.status = RunStatus::Voting;
16254            runner.execute_graph().await.unwrap();
16255            assert_eq!(runner.state.status, RunStatus::Blocked);
16256            assert!(!crate::handover::can_resume(&runner.state));
16257            assert_eq!(
16258                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16259                head
16260            );
16261        }
16262    }
16263
16264    #[cfg(unix)]
16265    #[tokio::test]
16266    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
16267        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16268        runner.state.authorize_base_sync(&task).unwrap();
16269        assert!(runner.state.consume_base_recovery(&head));
16270        runner.state.save().unwrap();
16271        runner.state = RunState::load(&runner.state.id).unwrap();
16272        runner.execute_graph().await.unwrap();
16273        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16274        assert_eq!(runner.state.status, RunStatus::Blocked);
16275        assert!(!runner.state.consume_base_recovery(&head));
16276        assert!(!crate::handover::can_resume(&runner.state));
16277        runner.state.authorize_base_sync(&task).unwrap();
16278        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
16279        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16280        assert_eq!(
16281            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16282            head
16283        );
16284    }
16285
16286    #[cfg(unix)]
16287    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
16288        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
16289        // This is an ordinary gate stop, not an authorized base-budget recovery.
16290        runner.state.prep_complete = Some(true);
16291        runner.state.base_sync = None;
16292        runner.state.base_sync_recovery = Default::default();
16293        runner.state.config.graph.gate_fix_rounds = 1;
16294        let mut review = review_round(true, 0, 1, 1, false, true);
16295        review.head = head.clone();
16296        review.verified_head = Some(head.clone());
16297        runner.state.reviews = vec![review];
16298        runner.state.gate = vec![CommandOutcome {
16299            command: "old gate".into(),
16300            code: Some(1),
16301            output_tail: "old failure".into(),
16302            duration_ms: 0,
16303            resource_blocked: false,
16304        }];
16305        runner.state.gate_head = Some(head.clone());
16306        runner.state.gate_ran = true;
16307        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
16308            agent: "fixture".into(),
16309            failed: runner.state.gate.clone(),
16310            notes: "spent".into(),
16311            committed: false,
16312            error: None,
16313        }];
16314        runner.state.land_approval = Some(crate::run::LandApproval {
16315            question: "old approval".into(),
16316            head: head.clone(),
16317        });
16318        runner.state.status = RunStatus::Blocked;
16319        (tmp, runner, head)
16320    }
16321
16322    #[cfg(unix)]
16323    #[tokio::test]
16324    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
16325        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
16326            let (tmp, mut runner, old) = resumed_gate_fixture().await;
16327            let phases = tmp.path().join("phases");
16328            runner.state.config.verify.e2e = vec![format!(
16329                "echo e2e >> '{}'; {}",
16330                phases.display(),
16331                if e2e_ok {
16332                    "true"
16333                } else {
16334                    "echo new-e2e-failure; false"
16335                }
16336            )];
16337            runner.state.config.verify.gate = vec![format!(
16338                "echo gate >> '{}'; {}",
16339                phases.display(),
16340                if gate_ok {
16341                    "true"
16342                } else {
16343                    "echo new-gate-failure; false"
16344                }
16345            )];
16346            runner.state.save().unwrap();
16347            runner.state = RunState::load(&runner.state.id).unwrap();
16348            runner.sync_to_base().await.unwrap();
16349            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16350            assert_ne!(head, old);
16351            // Reentry after Git updated but before verification was invalidated.
16352            runner.state = RunState::load(&runner.state.id).unwrap();
16353            runner.execute_graph().await.unwrap();
16354            assert_eq!(runner.state.gate_fixes.len(), 1);
16355            assert_eq!(runner.state.reviews.len(), 1);
16356            assert_eq!(runner.state.reviews[0].head, old);
16357            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
16358            assert_eq!(runner.state.verification_history.len(), 1);
16359            let history = &runner.state.verification_history[0];
16360            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
16361            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
16362            assert!(history.archived_at.is_some());
16363            assert_eq!(history.fixes_spent, Some(1));
16364            assert_eq!(history.gate[0].output_tail, "old failure");
16365            let archive_events: Vec<_> = runner
16366                .state
16367                .events
16368                .iter()
16369                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
16370                .collect();
16371            assert_eq!(archive_events.len(), 1);
16372            assert!(archive_events[0].message.contains(&short(&old)));
16373            assert!(archive_events[0].message.contains(&short(&head)));
16374            assert!(
16375                archive_events[0]
16376                    .message
16377                    .contains("1 gate-fix round(s) already spent stay spent")
16378            );
16379            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
16380            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
16381            if e2e_ok {
16382                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16383                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
16384            } else {
16385                assert!(!runner.state.gate_ran);
16386            }
16387            assert_eq!(
16388                runner.state.status,
16389                if e2e_ok && gate_ok {
16390                    RunStatus::Ready
16391                } else {
16392                    RunStatus::Blocked
16393                }
16394            );
16395            if !e2e_ok || !gate_ok {
16396                let cause = runner.state.verification_blocking_cause().unwrap();
16397                assert!(cause.contains(&head));
16398                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
16399                // A later successful progress event must not become the cause.
16400                runner.state.event("land", "rebased successfully");
16401                let notice = crate::notices::run_ended(&runner.state).unwrap();
16402                assert!(!format!("{notice:?}").contains("rebased successfully"));
16403                assert!(runner.state.merge.is_none());
16404            }
16405            let before = std::fs::read_to_string(&phases).unwrap();
16406            runner.state.save().unwrap();
16407            runner.state = RunState::load(&runner.state.id).unwrap();
16408            runner.review_loop().await.unwrap();
16409            runner.gate().await.unwrap();
16410            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
16411            assert_eq!(runner.state.verification_history.len(), 1);
16412            let history = &runner.state.verification_history[0];
16413            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
16414            assert!(history.archived_at.is_some());
16415            assert_eq!(history.fixes_spent, Some(1));
16416        }
16417    }
16418
16419    #[cfg(unix)]
16420    #[tokio::test]
16421    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
16422        let (tmp, mut runner, head) = resumed_gate_fixture().await;
16423        let marker = tmp.path().join("gate-ran");
16424        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
16425        runner.review_loop().await.unwrap();
16426        runner.gate().await.unwrap();
16427        assert_eq!(runner.state.status, RunStatus::Blocked);
16428        assert!(!marker.exists());
16429        assert!(runner.state.verification_history.is_empty());
16430        runner.state.gate_head = None;
16431        runner.gate().await.unwrap();
16432        assert!(marker.exists());
16433        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16434        assert_eq!(runner.state.verification_history.len(), 1);
16435        assert_eq!(runner.state.gate_fixes.len(), 1);
16436    }
16437
16438    #[cfg(unix)]
16439    #[tokio::test]
16440    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
16441        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16442        runner.sync_to_base().await.unwrap();
16443        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16444        runner.state.reconcile_verification_head(&head);
16445        // Both crash-before-command and resource wait are unresolved evidence.
16446        for outcomes in [
16447            Vec::new(),
16448            vec![CommandOutcome {
16449                command: "true".into(),
16450                code: None,
16451                output_tail: "cache unavailable".into(),
16452                duration_ms: 0,
16453                resource_blocked: true,
16454            }],
16455        ] {
16456            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16457                head: head.clone(),
16458                passed: false,
16459                outcomes,
16460            });
16461            runner.state.gate_ran = false;
16462            runner.state.gate.clear();
16463            runner.state.save().unwrap();
16464            runner.state = RunState::load(&runner.state.id).unwrap();
16465            runner.merge().await.unwrap();
16466            assert!(runner.state.merge.is_none());
16467            runner.review_loop().await.unwrap();
16468            runner.gate().await.unwrap();
16469            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
16470            assert!(runner.state.gate_allows_progress());
16471            assert_eq!(runner.state.gate_fixes.len(), 1);
16472            assert!(runner.state.merge.is_none());
16473        }
16474        // Moving the winner after a successful gate cannot publish an unchecked SHA.
16475        git::git(
16476            &runner.state.repo,
16477            &["commit", "--allow-empty", "-m", "external update"],
16478        )
16479        .await
16480        .unwrap();
16481        runner.merge().await.unwrap();
16482        assert!(runner.state.merge.is_none());
16483        assert_eq!(runner.state.status, RunStatus::Blocked);
16484    }
16485
16486    #[cfg(unix)]
16487    #[tokio::test]
16488    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
16489        for waiting_on_gate in [false, true] {
16490            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
16491            runner.sync_to_base().await.unwrap();
16492            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16493            runner.state.reconcile_verification_head(&head);
16494            if waiting_on_gate {
16495                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16496                    head: head.clone(),
16497                    passed: true,
16498                    outcomes: Vec::new(),
16499                });
16500            }
16501            let cache = tmp.path().join("cache");
16502            runner.state.config.verify.e2e =
16503                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
16504            runner.state.config.verify.gate = vec!["true".into()];
16505            runner.state.config.graph.timeout_resource_wait = 1;
16506            let owner =
16507                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
16508            let guard =
16509                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
16510                    crate::cache::AcquireOutcome::Acquired(g) => g,
16511                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
16512                };
16513            runner.review_loop().await.unwrap();
16514            runner.gate().await.unwrap();
16515            assert!(!runner.state.gate_ran);
16516            assert_eq!(runner.state.gate_fixes.len(), 1);
16517            assert!(runner.state.resource_block.is_some());
16518            runner.merge().await.unwrap();
16519            assert!(runner.state.merge.is_none());
16520            drop(guard);
16521            runner.state = RunState::load(&runner.state.id).unwrap();
16522            // The driver clears the previous pass's resource stop on resume.
16523            runner.state.resource_block = None;
16524            runner.review_loop().await.unwrap();
16525            runner.gate().await.unwrap();
16526            assert!(runner.state.gate_allows_progress());
16527            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16528            assert_eq!(runner.state.gate_fixes.len(), 1);
16529        }
16530    }
16531
16532    #[cfg(unix)]
16533    #[tokio::test]
16534    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
16535        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16536        runner.sync_to_base().await.unwrap();
16537        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16538        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
16539        // Verification passes before pre_gate and fails on the mechanically changed head.
16540        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
16541        runner.review_loop().await.unwrap();
16542        runner.gate().await.unwrap();
16543        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16544        assert_ne!(before, after);
16545        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
16546        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
16547        assert!(!runner.state.gate_allows_progress());
16548        runner.merge().await.unwrap();
16549        assert!(runner.state.merge.is_none());
16550        assert!(
16551            runner
16552                .state
16553                .verification_history
16554                .iter()
16555                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
16556        );
16557
16558        // An incomplete review is still an independent hold on a changed head.
16559        runner.state.reviews[0].clean = false;
16560        runner.state.reviews[0].answered = 0;
16561        runner.state.config.verify.e2e = vec!["true".into()];
16562        runner.review_loop().await.unwrap();
16563        runner.gate().await.unwrap();
16564        runner.merge().await.unwrap();
16565        assert_eq!(runner.state.status, RunStatus::Blocked);
16566        assert!(runner.state.merge.is_none());
16567        assert!(
16568            runner
16569                .state
16570                .verification_blocking_cause()
16571                .unwrap()
16572                .contains("review blocked")
16573        );
16574    }
16575
16576    #[cfg(unix)]
16577    #[tokio::test]
16578    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
16579        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
16580        runner.state.gate_head = None;
16581        runner.state.config.verify.gate =
16582            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
16583        runner.review_loop().await.unwrap();
16584        runner.gate().await.unwrap();
16585        assert_ne!(
16586            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16587            old
16588        );
16589        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
16590        assert!(!runner.state.gate_allows_progress());
16591        assert_eq!(runner.state.status, RunStatus::Blocked);
16592        runner.merge().await.unwrap();
16593        assert!(runner.state.merge.is_none());
16594    }
16595
16596    #[cfg(unix)]
16597    #[tokio::test]
16598    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
16599        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16600        let mut old_review = review_round(true, 0, 1, 1, false, true);
16601        old_review.head = head.clone();
16602        old_review.verified_head = Some(head.clone());
16603        runner.state.reviews = vec![old_review];
16604        runner.state.gate = vec![CommandOutcome {
16605            command: "old gate".into(),
16606            code: None,
16607            output_tail: "timed out after 2400s".into(),
16608            duration_ms: 2_400_000,
16609            resource_blocked: false,
16610        }];
16611        runner.state.gate_ran = true;
16612        runner.state.gate_head = Some(head.clone());
16613        runner.state.land_approval = Some(crate::run::LandApproval {
16614            question: "old-head-approval".into(),
16615            head: head.clone(),
16616        });
16617        runner.state.authorize_base_sync(&task).unwrap();
16618        runner.sync_to_base().await.unwrap();
16619        assert!(!runner.state.base_sync_blocked());
16620        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16621        assert_ne!(head, fresh);
16622        assert!(runner.state.reviews.is_empty());
16623        assert!(!runner.state.gate_ran);
16624        let saved = &runner.state.base_sync_recovery.authorizations[0];
16625        assert_eq!(saved.reviews[0].head, head);
16626        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
16627        assert!(saved.gate_ran);
16628        assert!(!saved.gate[0].ok());
16629        runner.review_loop().await.unwrap();
16630        runner.gate().await.unwrap();
16631        assert_eq!(runner.state.reviews[0].head, fresh);
16632        assert_eq!(
16633            runner.state.reviews[0].verified_head.as_deref(),
16634            Some(fresh.as_str())
16635        );
16636        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
16637        assert!(runner.state.gate_ran);
16638        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16639        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
16640        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
16641        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16642        assert_eq!(
16643            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
16644            "retained change"
16645        );
16646    }
16647
16648    #[cfg(unix)]
16649    #[tokio::test]
16650    async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
16651        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16652        let queue = crate::queue::Queue::open();
16653        let mut owner = queue.get(&task).unwrap();
16654        owner.release_operator("test-resume", None);
16655        queue.put(&mut owner).unwrap();
16656        let generation = owner.resumption.as_ref().unwrap().generation;
16657        runner.state.task_instruction = Some(owner.instruction.clone());
16658        runner.state.driver_exited = true;
16659        runner.state.save().unwrap();
16660        let home = crate::run::home();
16661        let _claim = queue.claim(&task).unwrap();
16662        owner.resumption.as_mut().unwrap().at =
16663            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
16664        queue.put(&mut owner).unwrap();
16665        assert!(
16666            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16667                .await
16668                .is_err(),
16669            "an approval from before the stop grants nothing"
16670        );
16671        owner.resumption.as_mut().unwrap().at = Timestamp::now();
16672        queue.put(&mut owner).unwrap();
16673        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16674            .await
16675            .unwrap();
16676        assert!(
16677            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16678                .await
16679                .is_err()
16680        );
16681        runner.state = RunState::load(&runner.state.id).unwrap();
16682        assert_eq!(
16683            runner.state.base_sync_recovery.authorizations[0].generation,
16684            Some(generation)
16685        );
16686        runner.sync_to_base().await.unwrap();
16687        let repo = runner.state.repo.clone();
16688        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16689        git::git(&repo, &["checkout", "main"]).await.unwrap();
16690        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
16691        git::git(&repo, &["add", "."]).await.unwrap();
16692        git::git(&repo, &["commit", "-m", "move base again"])
16693            .await
16694            .unwrap();
16695        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16696        git::git(&repo, &["checkout", "fix/retained"])
16697            .await
16698            .unwrap();
16699        runner.finish_after_tally().await.unwrap();
16700        assert_eq!(runner.state.status, RunStatus::Blocked);
16701        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16702        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16703        assert!(!runner.state.base_recovery_pending());
16704        assert!(!crate::handover::can_resume(&runner.state));
16705        assert!(!runner.state.gate_ran);
16706        // Even replaying the generation with a later timestamp grants nothing.
16707        owner.resumption.as_mut().unwrap().at = Timestamp::now();
16708        queue.put(&mut owner).unwrap();
16709        assert!(
16710            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16711                .await
16712                .is_err()
16713        );
16714        runner.state = RunState::load(&runner.state.id).unwrap();
16715        runner.sync_to_base().await.unwrap();
16716        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16717        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
16718        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16719    }
16720
16721    #[cfg(unix)]
16722    #[tokio::test]
16723    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
16724        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
16725        runner.state.base_sync = None;
16726        runner.state.status = RunStatus::Blocked;
16727        let mut second = runner.state.candidates[0].clone();
16728        second.label = 'B';
16729        runner.state.candidates.push(second);
16730        assert!(runner.state.votes.is_empty());
16731        assert!(runner.state.deliberation.is_empty());
16732        runner.deliberate().await.unwrap();
16733        runner.vote().await.unwrap();
16734        assert_eq!(runner.state.status, RunStatus::Blocked);
16735        assert!(runner.state.votes.is_empty());
16736        assert!(runner.state.deliberation.is_empty());
16737    }
16738
16739    /// A minimal, git-free `Runner` at a given status — `settle_questions`
16740    /// reads nothing else off it.
16741    fn runner_at(status: RunStatus) -> Runner {
16742        let mut state = RunState::new(
16743            PathBuf::from("/nonexistent/repo"),
16744            "main".to_owned(),
16745            "deadbeef".to_owned(),
16746            "task".to_owned(),
16747            Config::default(),
16748        );
16749        // These fixtures share one home and save concurrently. Short clock
16750        // IDs can collide, making an ownership test read another test's run.
16751        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
16752        state.id = format!(
16753            "20000101-000000-{:04x}",
16754            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
16755        );
16756        state.status = status;
16757        Runner {
16758            state,
16759            roles: ResolvedRoles {
16760                implementers: Vec::new(),
16761                judges: Vec::new(),
16762                reviewers: Vec::new(),
16763                fixer: None,
16764                conductor: conductor(),
16765                implementer_roster: Vec::new(),
16766                judge_roster: Vec::new(),
16767                reviewer_roster: Vec::new(),
16768            },
16769            sem: Arc::new(Semaphore::new(1)),
16770            pause: Pause::new(),
16771            interrupt: Pause::new(),
16772        }
16773    }
16774
16775    #[tokio::test]
16776    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
16777        crate::run::pin_test_home();
16778        let mut runner = runner_at(RunStatus::Gating);
16779        runner.state.save().unwrap();
16780        let id = runner.state.id.clone();
16781        let passes = runner.state.passes.len();
16782        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16783        let mut claiming = Box::pin(runner.claim_driver());
16784        std::future::poll_fn(|cx| {
16785            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
16786            std::task::Poll::Ready(())
16787        })
16788        .await;
16789        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
16790        drop(owner);
16791        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
16792            .await
16793            .unwrap()
16794            .unwrap();
16795        assert_eq!(execution.runner.state.passes.len(), passes + 1);
16796        assert!(execution.runner.state.resource_block.is_none());
16797        assert_eq!(
16798            execution.runner.state.liveness(false),
16799            crate::run::Liveness::Live
16800        );
16801        drop(execution);
16802        assert!(RunState::load(&id).unwrap().driver_exited);
16803    }
16804
16805    #[tokio::test]
16806    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
16807        crate::run::pin_test_home();
16808        let mut runner = runner_at(RunStatus::Gating);
16809        runner.state.save().unwrap();
16810        // run.json can commit, but the subsequent lease metadata write fails.
16811        // Cleanup must run after releasing the short ownership lock.
16812        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
16813        assert!(runner.claim_driver().await.is_err());
16814        assert!(runner.state.driver_exited);
16815        assert_eq!(runner.state.driver_pid, None);
16816        let disk = RunState::load(&runner.state.id).unwrap();
16817        assert!(disk.driver_exited);
16818        assert_eq!(disk.driver_pid, None);
16819        assert_eq!(disk.status, RunStatus::Gating);
16820    }
16821
16822    #[tokio::test]
16823    async fn an_early_graph_error_releases_its_driver() {
16824        crate::run::pin_test_home();
16825        let mut runner = runner_at(RunStatus::Blocked);
16826        runner.state.config.graph.selected_implementer = Some("selected".into());
16827        runner.state.save().unwrap();
16828        assert!(runner.execute().await.is_err());
16829        let disk = RunState::load(&runner.state.id).unwrap();
16830        assert!(disk.driver_exited);
16831        assert_eq!(disk.driver_pid, None);
16832        assert_eq!(disk.status, RunStatus::Blocked);
16833    }
16834
16835    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
16836        crate::run::MergeRepair {
16837            question: "repair-question".into(),
16838            head: "old-head".into(),
16839            pr: "https://github.com/o/r/pull/42".into(),
16840            reviews: vec![review_round(false, 1, 1, 1, false, true)],
16841            gate: vec![outcome(Some(0), false)],
16842            gate_head: Some("old-head".into()),
16843            gate_fixes: Vec::new(),
16844            pre_gate: Vec::new(),
16845            phase,
16846            failure: None,
16847        }
16848    }
16849
16850    #[tokio::test]
16851    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
16852        crate::run::pin_test_home();
16853        let mut runner = runner_at(RunStatus::Landing);
16854        runner.state.merge_repairs.push(merge_repair_request(
16855            crate::run::MergeRepairPhase::Requested,
16856        ));
16857        runner.state.save().unwrap();
16858        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
16859        assert!(
16860            runner
16861                .execute()
16862                .await
16863                .unwrap_err()
16864                .to_string()
16865                .contains("already running")
16866        );
16867        assert_eq!(
16868            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16869            crate::run::MergeRepairPhase::Requested
16870        );
16871        drop(claim);
16872        let mut disk = runner.state.clone();
16873        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
16874        disk.save().unwrap();
16875        assert!(
16876            runner
16877                .execute()
16878                .await
16879                .unwrap_err()
16880                .to_string()
16881                .contains("decision changed")
16882        );
16883        assert_eq!(
16884            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16885            crate::run::MergeRepairPhase::Complete
16886        );
16887    }
16888
16889    #[test]
16890    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
16891        crate::run::pin_test_home();
16892        let mut runner = runner_at(RunStatus::Landing);
16893        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
16894        runner.state.reviews = request.reviews.clone();
16895        runner.state.gate = request.gate.clone();
16896        runner.state.gate_ran = true;
16897        runner.state.land_approval = Some(crate::run::LandApproval {
16898            question: "old".into(),
16899            head: "old-head".into(),
16900        });
16901        runner.state.merge_repairs.push(request);
16902        runner.begin_merge_repair_review(0).unwrap();
16903        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
16904        assert!(!runner.state.gate_status().ok());
16905        assert!(runner.state.land_approval.is_none());
16906        let disk = RunState::load(&runner.state.id).unwrap();
16907        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
16908        assert!(disk.merge_repairs[0].gate[0].ok());
16909        assert_eq!(
16910            disk.merge_repairs[0].phase,
16911            crate::run::MergeRepairPhase::Reviewing
16912        );
16913        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
16914        assert!(disk.reviews.is_empty());
16915    }
16916
16917    #[tokio::test]
16918    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
16919        crate::run::pin_test_home();
16920        for phase in [
16921            crate::run::MergeRepairPhase::Fixing,
16922            crate::run::MergeRepairPhase::Requested,
16923        ] {
16924            let mut runner = runner_at(RunStatus::Landing);
16925            runner.state.config.graph.land_rounds = 0;
16926            runner.state.merge_repairs.push(merge_repair_request(phase));
16927            runner.state.save().unwrap();
16928            assert!(!runner.repair_merge_review().await.unwrap());
16929            assert_eq!(runner.state.status, RunStatus::Blocked);
16930            assert_eq!(
16931                runner.state.merge_repairs[0].phase,
16932                crate::run::MergeRepairPhase::Failed
16933            );
16934            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
16935            assert!(
16936                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
16937                    "interrupted"
16938                } else {
16939                    "budget exhausted"
16940                })
16941            );
16942            assert!(runner.state.merge.is_none());
16943        }
16944    }
16945
16946    /// `park_here` folding in the reason `Pause::park_because` recorded -
16947    /// this is what lets an operator reading a run's events tell an
16948    /// interrupt-driven park from an ordinary shutdown park.
16949    #[test]
16950    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
16951        crate::run::pin_test_home();
16952        let mut runner = runner_at(RunStatus::Implementing);
16953        let interrupt = Pause::new();
16954        runner.watch_interrupt(interrupt.clone());
16955
16956        interrupt.park_because("task a1b2 asked to run first");
16957
16958        assert!(runner.park_here().expect("park_here"));
16959        assert!(runner.state.parked);
16960        let last = runner.state.events.last().expect("a park event");
16961        assert_eq!(last.node, "park");
16962        assert!(
16963            last.message.contains("task a1b2 asked to run first"),
16964            "expected the interrupt reason in {:?}",
16965            last.message
16966        );
16967    }
16968
16969    /// The cause is stored on the pass when it parks, and the first cause a
16970    /// shared pause was given is the one that stays.
16971    #[test]
16972    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
16973        crate::run::pin_test_home();
16974        let mut runner = runner_at(RunStatus::Implementing);
16975        runner.state.begin_pass();
16976        let shutdown = Pause::new();
16977        runner.on_pause(shutdown.clone());
16978        let upgrade = ParkCause::Upgrade {
16979            from: "0.121.0".to_owned(),
16980            to: Some("0.121.1".to_owned()),
16981        };
16982        shutdown.park_for(upgrade.clone());
16983        shutdown.park_for(ParkCause::Operator);
16984
16985        assert!(runner.park_here().expect("park_here"));
16986        assert_eq!(
16987            runner.state.passes[0].end,
16988            Some(crate::run::PassEnd::Parked { cause: upgrade })
16989        );
16990        let last = runner.state.events.last().expect("a park event");
16991        assert!(last.message.contains("0.121.0"), "{}", last.message);
16992    }
16993
16994    #[tokio::test]
16995    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
16996        crate::run::pin_test_home();
16997        let mut runner = runner_at(RunStatus::Reviewing);
16998        runner.state.save().unwrap();
16999        let mut execution = runner.claim_driver().await.expect("first execution");
17000        assert_eq!(execution.runner.state.passes.len(), 1);
17001        execution
17002            .driver
17003            .finish(&mut execution.runner.state)
17004            .await
17005            .unwrap();
17006    }
17007
17008    #[tokio::test]
17009    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
17010        crate::run::pin_test_home();
17011        let mut runner = runner_at(RunStatus::Reviewing);
17012        runner.state.config.agents = vec![conductor()];
17013        runner.state.config.graph.timeout_resource_wait = 0;
17014        runner.state.save().unwrap();
17015        let id = runner.state.id.clone();
17016        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
17017        assert!(runner.claim_driver().await.is_err());
17018        drop(owner);
17019        drop(runner);
17020        let mut resumed = Runner::resume(&id).unwrap();
17021        assert_eq!(resumed.state.driver_claimed, Some(false));
17022        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
17023        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
17024        let mut execution = resumed
17025            .claim_driver()
17026            .await
17027            .expect("retry initial claim after reload");
17028        assert_eq!(execution.runner.state.driver_claimed, Some(true));
17029        assert_eq!(execution.runner.state.passes.len(), 1);
17030        execution
17031            .driver
17032            .finish(&mut execution.runner.state)
17033            .await
17034            .unwrap();
17035    }
17036
17037    #[tokio::test]
17038    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
17039        crate::run::pin_test_home();
17040        let mut runner = runner_at(RunStatus::Reviewing);
17041        runner.state.config.agents = vec![conductor()];
17042        runner.state.save().unwrap();
17043        let id = runner.state.id.clone();
17044        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
17045        let mut resumed = Runner::resume(&id).unwrap();
17046        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
17047        assert!(resumed.claim_driver().await.is_err());
17048        assert!(RunState::load(&id).unwrap().passes.is_empty());
17049        drop(lease);
17050        let mut execution = resumed.claim_driver().await.unwrap();
17051        execution
17052            .driver
17053            .finish(&mut execution.runner.state)
17054            .await
17055            .unwrap();
17056    }
17057
17058    #[tokio::test]
17059    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
17060        crate::run::pin_test_home();
17061        let mut runner = runner_at(RunStatus::Reviewing);
17062        let state = &mut runner.state;
17063        state.config.agents = vec![conductor()];
17064        state.driver_claimed = None;
17065        state.driver_pid = Some(std::process::id());
17066        state.driver_started_at = None;
17067        state.begin_pass();
17068        state.save().unwrap();
17069        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
17070        std::fs::write(
17071            state.dir().join("driver.json"),
17072            serde_json::to_vec(&serde_json::json!({
17073                "pid": state.driver_pid, "started_at": null,
17074                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
17075            }))
17076            .unwrap(),
17077        )
17078        .unwrap();
17079        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
17080        drop(lease);
17081        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
17082        let mut resumed = Runner::resume(&state.id).unwrap();
17083        assert!(!resumed.state.driver_exited);
17084        let mut execution = resumed
17085            .claim_driver()
17086            .await
17087            .expect("lease proves previous execution ended");
17088        assert_eq!(execution.runner.state.passes.len(), 2);
17089        execution
17090            .driver
17091            .finish(&mut execution.runner.state)
17092            .await
17093            .unwrap();
17094    }
17095
17096    #[tokio::test]
17097    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
17098        crate::run::pin_test_home();
17099        let mut runner = runner_at(RunStatus::Reviewing);
17100        let mut json = serde_json::to_value(&runner.state).unwrap();
17101        json.as_object_mut().unwrap().remove("driver_claimed");
17102        runner.state = serde_json::from_value(json).unwrap();
17103        assert_eq!(runner.state.driver_claimed, None);
17104        runner.state.save().unwrap();
17105        let mut execution = runner
17106            .claim_driver()
17107            .await
17108            .expect("the execution lease, not a missing marker, excludes a second driver");
17109        assert_eq!(execution.runner.state.passes.len(), 1);
17110        execution
17111            .driver
17112            .finish(&mut execution.runner.state)
17113            .await
17114            .unwrap();
17115    }
17116
17117    #[tokio::test]
17118    async fn driver_admission_refuses_a_proven_live_driver() {
17119        crate::run::pin_test_home();
17120        let mut runner = runner_at(RunStatus::Reviewing);
17121        let pid = std::process::id();
17122        runner.state.driver_pid = Some(pid);
17123        runner.state.driver_started_at = crate::proc::process_started_at(pid);
17124        assert!(runner.state.driver_started_at.is_some());
17125        runner.state.save().unwrap();
17126        let error = runner
17127            .claim_driver()
17128            .await
17129            .err()
17130            .expect("a live driver must refuse");
17131        assert!(error.to_string().contains("live driver"), "{error:#}");
17132        let saved = RunState::load(&runner.state.id).unwrap();
17133        assert_eq!(saved.driver_pid, Some(pid));
17134        assert_eq!(saved.status, RunStatus::Reviewing);
17135        assert!(saved.passes.is_empty());
17136    }
17137
17138    #[test]
17139    fn parking_resource_halt_persists_the_pass_once() {
17140        crate::run::pin_test_home();
17141        let mut runner = runner_at(RunStatus::Reviewing);
17142        runner.state.begin_pass();
17143        runner.state.resource_block = Some(crate::run::ResourceBlock {
17144            node: "fix".into(),
17145            kind: "shared build cache".into(),
17146            detail: "cancelled".into(),
17147            at: Timestamp::now(),
17148            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
17149        });
17150        let cause = ParkCause::Upgrade {
17151            from: "old".into(),
17152            to: Some("new".into()),
17153        };
17154        runner.pause.park_for(cause.clone());
17155        // Both direct fixer exits and graph resource boundaries must complete
17156        // parking, rather than returning a stale Reviewing record.
17157        runner.stop_for_resource().unwrap();
17158        assert!(runner.resource_halt().unwrap());
17159        assert!(runner.park_here().unwrap());
17160        let saved = RunState::load(&runner.state.id).unwrap();
17161        assert!(saved.parked);
17162        assert_eq!(saved.status, RunStatus::Reviewing);
17163        assert!(saved.passes[0].ended_at.is_some());
17164        assert_eq!(
17165            saved.passes[0].end,
17166            Some(crate::run::PassEnd::Parked {
17167                cause: cause.clone()
17168            })
17169        );
17170        assert_eq!(
17171            saved.resource_block.unwrap().cause,
17172            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
17173        );
17174        assert_eq!(
17175            saved
17176                .events
17177                .iter()
17178                .filter(|event| event.node == "park")
17179                .count(),
17180            1
17181        );
17182    }
17183
17184    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
17185    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
17186    /// cleared) must not make a *different* run - one only watching its own,
17187    /// unshared interrupt `Pause` - see itself as parked. If a future change
17188    /// ever collapsed these back into one handle, the interrupt scheduler
17189    /// would park every run for the rest of the daemon's life, not just the
17190    /// one it meant to interrupt.
17191    #[test]
17192    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
17193        crate::run::pin_test_home();
17194        let mut runner = runner_at(RunStatus::Implementing);
17195        let shutdown = Pause::new();
17196        runner.on_pause(shutdown.clone());
17197        let interrupt = Pause::new();
17198        runner.watch_interrupt(interrupt.clone());
17199
17200        // Nobody has asked for anything yet.
17201        assert!(!runner.park_here().expect("park_here"));
17202        assert!(!runner.state.parked);
17203
17204        // Only the interrupt handle fires; the shutdown handle stays clear.
17205        interrupt.park_because("test");
17206        assert!(!shutdown.parked());
17207        assert!(runner.park_here().expect("park_here"));
17208    }
17209
17210    /// The property every prior attempt at this feature failed to pin down:
17211    /// asking a run to park while one of its nodes has a real, in-flight
17212    /// async operation running (an agent call, in production) must not cut
17213    /// that operation short. `park_here` is only ever consulted *between*
17214    /// `execute`'s node calls - see its own doc - so nothing inside a node
17215    /// can observe a park request until the node itself returns. This proves
17216    /// that structurally, with real `tokio` concurrency and a channel
17217    /// handshake (never a sleep, which would only prove "usually", not
17218    /// "cannot"): the "node" below reports that it has genuinely started,
17219    /// and only then is the park requested; the node still has to be told to
17220    /// finish before `park_here` is ever called, exactly mirroring every
17221    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
17222    /// in `execute`.
17223    #[tokio::test]
17224    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
17225        crate::run::pin_test_home();
17226        let mut runner = runner_at(RunStatus::Implementing);
17227        let interrupt = Pause::new();
17228        runner.watch_interrupt(interrupt.clone());
17229
17230        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
17231        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
17232
17233        // Stands in for one node's in-flight agent call: it proves it has
17234        // genuinely started, then blocks - exactly as a spawned CLI process
17235        // does - until told to finish.
17236        let node = async move {
17237            started_tx.send(()).expect("send started");
17238            finish_rx.await.expect("recv finish");
17239            "node finished"
17240        };
17241
17242        let interrupter = async move {
17243            started_rx.await.expect("recv started");
17244            // The call is now genuinely in flight. Ask it to park.
17245            interrupt.park_because("higher-priority task waiting");
17246            // Nothing the node does can observe this yet - there is no
17247            // check inside it, by construction - so let the executor run
17248            // anything pending and then let the node finish on its own.
17249            tokio::task::yield_now().await;
17250            finish_tx.send(()).expect("send finish");
17251        };
17252
17253        let (node_result, ()) = tokio::join!(node, interrupter);
17254        assert_eq!(
17255            node_result, "node finished",
17256            "the in-flight call ran to completion"
17257        );
17258
17259        // Only now, at the boundary the real `execute` would check right
17260        // after this node, does the park take effect.
17261        assert!(runner.park_here().expect("park_here"));
17262        assert!(runner.state.parked);
17263    }
17264
17265    /// A run parked mid-competition carries every field it had accumulated
17266    /// through the exact same disk round-trip an ordinary resume uses -
17267    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
17268    /// Nothing about parking for an interrupt is a special case of that path;
17269    /// this is what proves it rather than assuming it.
17270    #[test]
17271    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
17272        crate::run::pin_test_home();
17273        let mut runner = runner_at(RunStatus::Judging);
17274        // `Runner::resume` re-resolves roles from the saved config, which
17275        // refuses an empty roster - give it the same minimal one `conductor`
17276        // itself uses.
17277        runner.state.config.agents = vec![conductor()];
17278        runner.state.candidates = vec![Candidate {
17279            index: 0,
17280            label: 'A',
17281            agent: "alpha".to_owned(),
17282            branch: "magi/x/A".to_owned(),
17283            worktree: PathBuf::from("/nonexistent/worktree"),
17284            summary: "did the thing".to_owned(),
17285            stat: "1 file changed".to_owned(),
17286            files: 1,
17287            commits: 1,
17288            empty: false,
17289            failed: None,
17290            verified_noop: None,
17291            duration_ms: 1234,
17292            folded: false,
17293        }];
17294        let run_id = runner.state.id.clone();
17295
17296        let interrupt = Pause::new();
17297        runner.watch_interrupt(interrupt.clone());
17298        interrupt.park_because("task c3d4 asked to run first");
17299        assert!(runner.park_here().expect("park_here"));
17300
17301        let resumed = Runner::resume(&run_id).expect("resume");
17302        assert_eq!(resumed.state.candidates.len(), 1);
17303        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
17304        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
17305        assert_eq!(resumed.state.status, runner.state.status);
17306        assert!(
17307            resumed.state.parked,
17308            "still parked until `execute` actually walks the graph again"
17309        );
17310        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
17311    }
17312
17313    /// A fresh open question on `run`, stored and handed back for assertions.
17314    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
17315        let mut q = ask::Question::new(
17316            run.to_owned(),
17317            "implement".to_owned(),
17318            "impl-A".to_owned(),
17319            "Which storage backend should the cache use?".to_owned(),
17320            String::new(),
17321            vec!["SQLite".to_owned(), "Redis".to_owned()],
17322        );
17323        store.put(&mut q).unwrap();
17324        q
17325    }
17326
17327    #[test]
17328    fn a_failed_runs_open_question_is_abandoned() {
17329        ask_test_home();
17330        let store = ask::Questions::open();
17331        let mut runner = runner_at(RunStatus::Failed);
17332        let run = runner.state.id.clone();
17333        let q = ask_open_question(&store, &run);
17334
17335        runner.settle_questions();
17336
17337        let back = store.get(&q.id).unwrap();
17338        assert!(
17339            !back.status.open(),
17340            "the seat that asked died with the run; nobody is left to read an answer"
17341        );
17342        assert!(
17343            back.detail.contains(&run) && back.detail.contains("failed"),
17344            "the reason names what the run became, not just that it is gone: {}",
17345            back.detail
17346        );
17347    }
17348
17349    #[test]
17350    fn a_merged_runs_open_question_is_abandoned_too() {
17351        ask_test_home();
17352        let store = ask::Questions::open();
17353        // A run that finishes cleanly still leaves nobody to read an answer -
17354        // this is not only a failure-path cleanup.
17355        for status in [RunStatus::Merged, RunStatus::Ready] {
17356            let mut runner = runner_at(status);
17357            let run = runner.state.id.clone();
17358            let q = ask_open_question(&store, &run);
17359
17360            runner.settle_questions();
17361
17362            let back = store.get(&q.id).unwrap();
17363            assert!(
17364                !back.status.open(),
17365                "{status:?} run's question must not outlive the run"
17366            );
17367        }
17368    }
17369
17370    #[test]
17371    fn a_still_resumable_runs_open_question_is_left_alone() {
17372        ask_test_home();
17373        let store = ask::Questions::open();
17374        // `Blocked` and `Stalled` can still be resumed — the candidates, the
17375        // review round and the seat sessions are all still on disk — so a
17376        // question asked mid-round may yet get a real answer from a real
17377        // resume. Sweeping it here would be exactly the failure mode this
17378        // whole feature exists to avoid on the other side.
17379        for status in [RunStatus::Blocked, RunStatus::Stalled] {
17380            let mut runner = runner_at(status);
17381            let run = runner.state.id.clone();
17382            let q = ask_open_question(&store, &run);
17383
17384            runner.settle_questions();
17385
17386            let back = store.get(&q.id).unwrap();
17387            assert!(
17388                back.status.open(),
17389                "{status:?} is still alive; the question must still be waiting"
17390            );
17391        }
17392    }
17393
17394    #[test]
17395    fn settle_questions_never_touches_an_already_answered_question() {
17396        ask_test_home();
17397        let store = ask::Questions::open();
17398        let mut runner = runner_at(RunStatus::Failed);
17399        let run = runner.state.id.clone();
17400        let mut q = ask_open_question(&store, &run);
17401        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
17402            .unwrap();
17403        store.put(&mut q).unwrap();
17404
17405        // Called twice, the way a crash-recovered daemon reclaim and the
17406        // graph's own cleanup both can for the same run — `abandon_for_run`
17407        // only ever touches what is still open, so this must be inert both
17408        // times, not merely the second.
17409        runner.settle_questions();
17410        runner.settle_questions();
17411
17412        let back = store.get(&q.id).unwrap();
17413        assert_eq!(
17414            back.status,
17415            ask::QuestionStatus::Answered,
17416            "a real answer is a decision on record, never overwritten by a sweep"
17417        );
17418    }
17419
17420    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
17421    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
17422    /// without merging, whose winner is still the operator's answer to read.
17423    /// Nothing previously called `fold_run` itself with a real `tally`, so
17424    /// this is the first test to pin down the one distinction the whole
17425    /// automatic-fold feature depends on: the winner's worktree and branch
17426    /// must survive, everything else sharing the run's worktree bay - a
17427    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
17428    /// second sweep treats every non-winner directory under the bay alike -
17429    /// must not.
17430    #[tokio::test]
17431    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
17432        crate::run::pin_test_home();
17433        let tmp = tempfile::tempdir().expect("tempdir");
17434        let repo = tmp.path().join("repo");
17435        std::fs::create_dir_all(&repo).unwrap();
17436        init_repo(&repo);
17437
17438        let mut config = Config::default();
17439        config.graph.worktree_root = Some(tmp.path().join("wt"));
17440
17441        let mut state = RunState::new(
17442            repo.clone(),
17443            "main".to_owned(),
17444            "deadbeef".to_owned(),
17445            "task".to_owned(),
17446            config,
17447        );
17448        let root = state.worktree_root();
17449        let wt_a = root.join("cand-A");
17450        let wt_b = root.join("cand-B");
17451        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
17452            .await
17453            .expect("worktree A");
17454        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
17455            .await
17456            .expect("worktree B");
17457
17458        state.candidates = vec![
17459            Candidate {
17460                index: 0,
17461                label: 'A',
17462                agent: "alpha".to_owned(),
17463                branch: "magi/x/A".to_owned(),
17464                worktree: wt_a.clone(),
17465                summary: String::new(),
17466                stat: String::new(),
17467                files: 0,
17468                commits: 0,
17469                empty: false,
17470                failed: None,
17471                verified_noop: None,
17472                duration_ms: 0,
17473                folded: false,
17474            },
17475            Candidate {
17476                index: 1,
17477                label: 'B',
17478                agent: "beta".to_owned(),
17479                branch: "magi/x/B".to_owned(),
17480                worktree: wt_b.clone(),
17481                summary: String::new(),
17482                stat: String::new(),
17483                files: 0,
17484                commits: 0,
17485                empty: false,
17486                failed: None,
17487                verified_noop: None,
17488                duration_ms: 0,
17489                folded: false,
17490            },
17491        ];
17492        state.tally = Some(Tally {
17493            first_choice: BTreeMap::from([('A', 1)]),
17494            borda: BTreeMap::new(),
17495            winner: 'A',
17496            rankings: 1,
17497            unanimous_initial: true,
17498            deliberated: false,
17499            changed_votes: 0,
17500            unanimous_final: true,
17501            tie_break: None,
17502            judges: 1,
17503            present: 1,
17504            quorum: 1,
17505            met_quorum: true,
17506            uncontested: None,
17507        });
17508        state.status = RunStatus::Ready;
17509
17510        fold_run(&mut state, false, &crate::run::home())
17511            .await
17512            .expect("fold_run");
17513
17514        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
17515        assert!(
17516            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17517            "the unmerged winner's branch survives"
17518        );
17519        assert!(
17520            !state.candidates[0].folded,
17521            "the winner is not marked folded"
17522        );
17523
17524        assert!(!wt_b.exists(), "the loser's worktree is removed");
17525        assert!(
17526            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
17527            "the loser's branch is removed"
17528        );
17529        assert!(state.candidates[1].folded, "the loser is marked folded");
17530    }
17531
17532    #[tokio::test]
17533    #[cfg(unix)]
17534    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
17535        use std::os::fd::AsRawFd as _;
17536        use std::time::Duration;
17537
17538        let tmp = tempfile::tempdir().unwrap();
17539        let repo = tmp.path().join("repo");
17540        std::fs::create_dir_all(&repo).unwrap();
17541        init_repo(&repo);
17542        let common = repo.join(".git");
17543        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
17544        let home = tmp.path().join("home");
17545        let mut config = Config::default();
17546        config.coordination = crate::config::test_coordination();
17547        config.graph.worktree_root = Some(tmp.path().join("wt"));
17548        config.graph.timeout_resource_wait = 1;
17549        let mut state = RunState::new(
17550            repo.clone(),
17551            "main".into(),
17552            "deadbeef".into(),
17553            "task".into(),
17554            config,
17555        );
17556        let root = state.worktree_root();
17557        let candidate = root.join("cand-A");
17558        let reviewer = root.join("review-1");
17559        let branch = state.branch_for('A');
17560        git::worktree_add_branch(&repo, &candidate, &branch, "main")
17561            .await
17562            .unwrap();
17563        git::worktree_add_detached(&repo, &reviewer, "main")
17564            .await
17565            .unwrap();
17566        git::acquire_worktree_config(&repo).await.unwrap();
17567        state.enabled_worktree_config = true;
17568        state.status = RunStatus::Ready;
17569        state.candidates.push(Candidate {
17570            index: 0,
17571            label: 'A',
17572            agent: "implement".into(),
17573            branch: branch.clone(),
17574            worktree: candidate.clone(),
17575            summary: String::new(),
17576            stat: String::new(),
17577            files: 0,
17578            commits: 0,
17579            empty: false,
17580            failed: None,
17581            verified_noop: None,
17582            duration_ms: 0,
17583            folded: false,
17584        });
17585        state.save_under(&home).unwrap();
17586        let config_before = std::fs::read(common.join("config")).unwrap();
17587        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
17588            .await
17589            .unwrap();
17590        // A separate OS handle bypasses the launcher's local gate, exactly as
17591        // an external updater does, and must still prevent all folding work.
17592        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
17593        assert_eq!(
17594            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
17595            0
17596        );
17597        let mut fold = Box::pin(fold_run(&mut state, true, &home));
17598        assert!(
17599            tokio::time::timeout(Duration::from_millis(200), &mut fold)
17600                .await
17601                .is_err()
17602        );
17603        assert!(candidate.exists());
17604        assert!(reviewer.exists());
17605        assert!(git::branch_exists(&repo, &branch).await.unwrap());
17606        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17607        assert_eq!(
17608            git::git(&repo, &["worktree", "list", "--porcelain"])
17609                .await
17610                .unwrap(),
17611            worktrees_before
17612        );
17613        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
17614            .await
17615            .unwrap()
17616            .unwrap_err();
17617        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
17618        drop(fold);
17619        assert!(candidate.exists() && reviewer.exists());
17620        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
17621        assert_eq!(
17622            state.resource_block.as_ref().unwrap().kind,
17623            "repository coordination"
17624        );
17625        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17626        drop(external);
17627        let mut fold = Box::pin(fold_run(&mut state, true, &home));
17628        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
17629            .await
17630            .unwrap()
17631            .unwrap();
17632        drop(fold);
17633        assert!(removed.contains(&branch));
17634        assert!(!root.exists());
17635        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
17636        assert!(state.candidates[0].folded);
17637        assert!(!state.enabled_worktree_config);
17638        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
17639        let enabled = git::git_raw(
17640            &repo,
17641            &["config", "--local", "--get", "extensions.worktreeConfig"],
17642        )
17643        .await
17644        .unwrap();
17645        assert!(!enabled.ok(), "the last config reference was released");
17646    }
17647
17648    /// A branch handed to a later run is that run's (and its pull request's):
17649    /// folding the run that released it must not delete it.
17650    #[tokio::test]
17651    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
17652        let tmp = tempfile::tempdir().expect("tempdir");
17653        let repo = tmp.path().join("repo");
17654        std::fs::create_dir_all(&repo).unwrap();
17655        init_repo(&repo);
17656        let home = tmp.path().join("home");
17657
17658        let mut config = Config::default();
17659        config.graph.worktree_root = Some(tmp.path().join("wt"));
17660        let mut state = RunState::new(
17661            repo.clone(),
17662            "main".to_owned(),
17663            "deadbeef".to_owned(),
17664            "task".to_owned(),
17665            config,
17666        );
17667        // The worktree is already gone (released); the branch survives.
17668        git::git(&repo, &["branch", "magi/x/A", "main"])
17669            .await
17670            .expect("branch");
17671        state.candidates = vec![Candidate {
17672            index: 0,
17673            label: 'A',
17674            agent: "alpha".to_owned(),
17675            branch: "magi/x/A".to_owned(),
17676            worktree: state.worktree_root().join("cand-A"),
17677            summary: String::new(),
17678            stat: String::new(),
17679            files: 0,
17680            commits: 0,
17681            empty: false,
17682            failed: None,
17683            verified_noop: None,
17684            duration_ms: 0,
17685            folded: true,
17686        }];
17687        state.released_to = Some("20260901-000000-new1".to_owned());
17688        state.released_branches = vec!["magi/x/A".to_owned()];
17689
17690        fold_run(&mut state, true, &home).await.expect("fold_run");
17691        state.status = RunStatus::Merged;
17692        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
17693        state.merge = Some(crate::run::MergeOutcome {
17694            mode: MergeMode::Pr,
17695            ok: true,
17696            detail: "confirmed merged".into(),
17697            empty: false,
17698        });
17699        fold_run(&mut state, true, &home)
17700            .await
17701            .expect("merged fold");
17702
17703        assert!(
17704            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17705            "the handed-over branch survives a fold"
17706        );
17707    }
17708
17709    /// A winner with nothing ahead of the base is caught before `gh` is ever
17710    /// asked for a pull request, and the message carries what the task's
17711    /// references resolved to.
17712    #[tokio::test]
17713    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
17714        let tmp = tempfile::tempdir().expect("tempdir");
17715        let repo = tmp.path().join("repo");
17716        std::fs::create_dir_all(&repo).unwrap();
17717        init_repo(&repo);
17718        let run = |args: &[&str]| {
17719            let out = std::process::Command::new("git")
17720                .quiet()
17721                .args(args)
17722                .current_dir(&repo)
17723                .output()
17724                .expect("spawn git");
17725            assert!(out.status.success(), "git {args:?}");
17726        };
17727        run(&["branch", "magi/x/A"]);
17728        run(&["checkout", "-q", "-b", "magi/x/B"]);
17729        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
17730        run(&["add", "-A"]);
17731        run(&["commit", "-q", "-m", "work"]);
17732        run(&["checkout", "-q", "main"]);
17733
17734        // A pull request is compared against the remote's base, so the
17735        // fixture needs one. Before it exists nothing can be read, and the
17736        // answer must be "not empty".
17737        let probe = RunState::new(
17738            repo.clone(),
17739            "main".to_owned(),
17740            "deadbeef".to_owned(),
17741            "task".to_owned(),
17742            Config::default(),
17743        );
17744        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
17745        let bare = tmp.path().join("origin.git");
17746        let out = std::process::Command::new("git")
17747            .quiet()
17748            .args(["init", "-q", "--bare"])
17749            .arg(&bare)
17750            .output()
17751            .expect("spawn git");
17752        assert!(out.status.success(), "git init --bare");
17753        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
17754        run(&["push", "-q", "origin", "main"]);
17755
17756        let mut state = RunState::new(
17757            repo.clone(),
17758            "main".to_owned(),
17759            "deadbeef".to_owned(),
17760            "task".to_owned(),
17761            Config::default(),
17762        );
17763        state.seeds = vec![refs::Seed {
17764            token: "magi/27b2/A".to_owned(),
17765            kind: refs::SeedKind::Unresolved,
17766            sha: String::new(),
17767            branch: true,
17768            detail: "no branch or commit named magi/27b2/A".to_owned(),
17769        }];
17770
17771        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
17772        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
17773        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
17774        let detail = empty_candidate_detail(&state, "main");
17775        assert!(detail.starts_with("empty candidate"), "{detail}");
17776        assert!(!detail.contains("magi/27b2/A"), "{detail}");
17777        assert!(detail.contains("--start-from"), "{detail}");
17778    }
17779
17780    /// `status == Ready` used to be read as "this is the harmless
17781    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
17782    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
17783    /// run's PR was closed without merging — and reentering `merge` with
17784    /// `mode` still `Pr` does not know the difference, so it pushed and
17785    /// opened a second pull request. `mode == Local` reproduces the same
17786    /// blind spot without a network call: reentry must not attempt another
17787    /// git merge once this node has already recorded an outcome.
17788    #[tokio::test]
17789    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
17790        let tmp = tempfile::tempdir().expect("tempdir");
17791        let repo = tmp.path().join("repo");
17792        std::fs::create_dir_all(&repo).unwrap();
17793        init_repo(&repo);
17794
17795        let mut config = Config::default();
17796        config.merge.mode = MergeMode::Local;
17797
17798        let mut state = RunState::new(
17799            repo.clone(),
17800            "main".to_owned(),
17801            "deadbeef".to_owned(),
17802            "task".to_owned(),
17803            config,
17804        );
17805        state.candidates = vec![Candidate {
17806            index: 0,
17807            label: 'A',
17808            agent: "alpha".to_owned(),
17809            branch: "does-not-exist".to_owned(),
17810            worktree: repo.clone(),
17811            summary: String::new(),
17812            stat: String::new(),
17813            files: 0,
17814            commits: 0,
17815            empty: false,
17816            failed: None,
17817            verified_noop: None,
17818            duration_ms: 0,
17819            folded: false,
17820        }];
17821        state.tally = Some(Tally {
17822            first_choice: BTreeMap::from([('A', 1)]),
17823            borda: BTreeMap::new(),
17824            winner: 'A',
17825            rankings: 1,
17826            unanimous_initial: true,
17827            deliberated: false,
17828            changed_votes: 0,
17829            unanimous_final: true,
17830            tie_break: None,
17831            judges: 0,
17832            present: 0,
17833            quorum: 0,
17834            met_quorum: true,
17835            uncontested: Some("only candidate A produced a change".to_owned()),
17836        });
17837        state.reviews = vec![ReviewRound {
17838            round: 1,
17839            head: "deadbeef".to_owned(),
17840            verified_head: None,
17841            verified_at: None,
17842            reviews: Vec::new(),
17843            e2e: Vec::new(),
17844            fix: None,
17845            blocking: 0,
17846            answered: 0,
17847            expected: 0,
17848            clean: true,
17849            verify_retried: false,
17850            e2e_deferred: false,
17851            e2e_defer_reason: None,
17852            progressed: false,
17853            vote_split: false,
17854            reconsideration: Vec::new(),
17855            verdict: None,
17856        }];
17857        state.gate = vec![CommandOutcome {
17858            command: "test".to_owned(),
17859            code: Some(0),
17860            output_tail: String::new(),
17861            duration_ms: 0,
17862            resource_blocked: false,
17863        }];
17864        state.gate_ran = true;
17865        // Reached its conclusion already — e.g. `land` closing the PR without
17866        // merging it, which (like the honest `MergeMode::None` path) leaves
17867        // `status` at `Ready`. The recorded outcome is what actually marks
17868        // this node done.
17869        state.status = RunStatus::Ready;
17870        state.merge = Some(MergeOutcome {
17871            mode: MergeMode::Local,
17872            ok: false,
17873            detail: "already concluded".to_owned(),
17874            empty: false,
17875        });
17876
17877        let mut runner = Runner {
17878            state,
17879            roles: ResolvedRoles {
17880                implementers: Vec::new(),
17881                judges: Vec::new(),
17882                reviewers: Vec::new(),
17883                fixer: None,
17884                conductor: conductor(),
17885                implementer_roster: Vec::new(),
17886                judge_roster: Vec::new(),
17887                reviewer_roster: Vec::new(),
17888            },
17889            sem: Arc::new(Semaphore::new(1)),
17890            pause: Pause::new(),
17891            interrupt: Pause::new(),
17892        };
17893
17894        runner.merge().await.expect("merge");
17895
17896        assert_eq!(
17897            runner.state.status,
17898            RunStatus::Ready,
17899            "a concluded run's status must not change on reentry"
17900        );
17901        assert_eq!(
17902            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17903            Some("already concluded"),
17904            "merge must not run again once the node already recorded an outcome"
17905        );
17906    }
17907
17908    /// `gate` leaves `state.gate_ran` false both before it has ever run and
17909    /// when its last attempt was resource-blocked (the shared build cache
17910    /// could not be acquired or confirmed fresh in time - see
17911    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
17912    /// `Vec` this also leaves behind used to read as "nothing failed" and let
17913    /// a run merge a tree the gate never actually checked - exactly the case
17914    /// a contended cache produces on every retry until it clears. `merge`
17915    /// must refuse until `gate` has actually recorded an attempt.
17916    #[tokio::test]
17917    async fn merge_refuses_a_gate_that_has_not_actually_run() {
17918        let tmp = tempfile::tempdir().expect("tempdir");
17919        let repo = tmp.path().join("repo");
17920        std::fs::create_dir_all(&repo).unwrap();
17921        init_repo(&repo);
17922
17923        let mut config = Config::default();
17924        config.merge.mode = MergeMode::Local;
17925
17926        let mut state = RunState::new(
17927            repo.clone(),
17928            "main".to_owned(),
17929            "deadbeef".to_owned(),
17930            "task".to_owned(),
17931            config,
17932        );
17933        state.candidates = vec![Candidate {
17934            index: 0,
17935            label: 'A',
17936            agent: "alpha".to_owned(),
17937            branch: "does-not-exist".to_owned(),
17938            worktree: repo.clone(),
17939            summary: String::new(),
17940            stat: String::new(),
17941            files: 0,
17942            commits: 0,
17943            empty: false,
17944            failed: None,
17945            verified_noop: None,
17946            duration_ms: 0,
17947            folded: false,
17948        }];
17949        state.tally = Some(Tally {
17950            first_choice: BTreeMap::from([('A', 1)]),
17951            borda: BTreeMap::new(),
17952            winner: 'A',
17953            rankings: 1,
17954            unanimous_initial: true,
17955            deliberated: false,
17956            changed_votes: 0,
17957            unanimous_final: true,
17958            tie_break: None,
17959            judges: 0,
17960            present: 0,
17961            quorum: 0,
17962            met_quorum: true,
17963            uncontested: Some("only candidate A produced a change".to_owned()),
17964        });
17965        state.reviews = vec![ReviewRound {
17966            round: 1,
17967            head: "deadbeef".to_owned(),
17968            verified_head: None,
17969            verified_at: None,
17970            reviews: Vec::new(),
17971            e2e: Vec::new(),
17972            fix: None,
17973            blocking: 0,
17974            answered: 0,
17975            expected: 0,
17976            clean: true,
17977            verify_retried: false,
17978            e2e_deferred: false,
17979            e2e_defer_reason: None,
17980            progressed: false,
17981            vote_split: false,
17982            reconsideration: Vec::new(),
17983            verdict: None,
17984        }];
17985        // The point: `gate` has not recorded anything yet.
17986        state.gate = Vec::new();
17987        state.gate_ran = false;
17988        state.status = RunStatus::Gating;
17989
17990        let mut runner = Runner {
17991            state,
17992            roles: ResolvedRoles {
17993                implementers: Vec::new(),
17994                judges: Vec::new(),
17995                reviewers: Vec::new(),
17996                fixer: None,
17997                conductor: conductor(),
17998                implementer_roster: Vec::new(),
17999                judge_roster: Vec::new(),
18000                reviewer_roster: Vec::new(),
18001            },
18002            sem: Arc::new(Semaphore::new(1)),
18003            pause: Pause::new(),
18004            interrupt: Pause::new(),
18005        };
18006
18007        runner.merge().await.expect("merge");
18008
18009        assert!(
18010            runner.state.merge.is_none(),
18011            "an empty gate must never be read as a passing one: {:?}",
18012            runner.state.merge
18013        );
18014    }
18015
18016    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
18017    /// commands configured and `merge.mode` is `none` (a review-only run).
18018    /// `gate` must still record a real attempt — zero commands, vacuously
18019    /// passed — rather than leaving `state.gate` empty in a way `merge`
18020    /// cannot tell apart from "never ran"; otherwise the run reaches
18021    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
18022    #[tokio::test]
18023    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
18024        let tmp = tempfile::tempdir().expect("tempdir");
18025        let repo = tmp.path().join("repo");
18026        std::fs::create_dir_all(&repo).unwrap();
18027        init_repo(&repo);
18028
18029        // Default config: `verify.gate` empty, `merge.mode` is `none`.
18030        let config = Config::default();
18031
18032        let mut state = RunState::new(
18033            repo.clone(),
18034            "main".to_owned(),
18035            "deadbeef".to_owned(),
18036            "task".to_owned(),
18037            config,
18038        );
18039        state.candidates = vec![Candidate {
18040            index: 0,
18041            label: 'A',
18042            agent: "alpha".to_owned(),
18043            branch: "does-not-exist".to_owned(),
18044            worktree: repo.clone(),
18045            summary: String::new(),
18046            stat: String::new(),
18047            files: 0,
18048            commits: 0,
18049            empty: false,
18050            failed: None,
18051            verified_noop: None,
18052            duration_ms: 0,
18053            folded: false,
18054        }];
18055        state.tally = Some(Tally {
18056            first_choice: BTreeMap::from([('A', 1)]),
18057            borda: BTreeMap::new(),
18058            winner: 'A',
18059            rankings: 1,
18060            unanimous_initial: true,
18061            deliberated: false,
18062            changed_votes: 0,
18063            unanimous_final: true,
18064            tie_break: None,
18065            judges: 0,
18066            present: 0,
18067            quorum: 0,
18068            met_quorum: true,
18069            uncontested: Some("only candidate A produced a change".to_owned()),
18070        });
18071        state.reviews = vec![ReviewRound {
18072            round: 1,
18073            head: "deadbeef".to_owned(),
18074            verified_head: None,
18075            verified_at: None,
18076            reviews: Vec::new(),
18077            e2e: Vec::new(),
18078            fix: None,
18079            blocking: 0,
18080            answered: 0,
18081            expected: 0,
18082            clean: true,
18083            verify_retried: false,
18084            e2e_deferred: false,
18085            e2e_defer_reason: None,
18086            progressed: false,
18087            vote_split: false,
18088            reconsideration: Vec::new(),
18089            verdict: None,
18090        }];
18091
18092        let mut runner = Runner {
18093            state,
18094            roles: ResolvedRoles {
18095                implementers: Vec::new(),
18096                judges: Vec::new(),
18097                reviewers: Vec::new(),
18098                fixer: None,
18099                conductor: conductor(),
18100                implementer_roster: Vec::new(),
18101                judge_roster: Vec::new(),
18102                reviewer_roster: Vec::new(),
18103            },
18104            sem: Arc::new(Semaphore::new(1)),
18105            pause: Pause::new(),
18106            interrupt: Pause::new(),
18107        };
18108
18109        runner.gate().await.expect("gate");
18110        assert!(
18111            runner.state.gate_ran,
18112            "zero configured commands is still a real attempt, not an unrun gate"
18113        );
18114        assert!(runner.state.gate.is_empty());
18115        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
18116        assert_ne!(
18117            runner.state.status,
18118            RunStatus::Blocked,
18119            "a gate with nothing to check must not read as failed"
18120        );
18121
18122        runner.merge().await.expect("merge");
18123        assert_eq!(
18124            runner.state.status,
18125            RunStatus::Ready,
18126            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
18127        );
18128    }
18129
18130    /// `Config::cache_dir` is derived from `verify.e2e` as well as
18131    /// `verify.gate` (so the e2e leg and the final gate never build against
18132    /// different directories). With zero `verify.gate` commands but a
18133    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
18134    /// that lease before discovering it had nothing to run - so a repo with
18135    /// no gate commands could come back `resource_blocked` (and therefore
18136    /// still `gate_ran == false`) on nothing but an unrelated run holding the
18137    /// cache, exactly the contention this run's own zero commands could
18138    /// never have touched. `gate` must recognise there is nothing to check
18139    /// before it ever asks for the lease.
18140    #[tokio::test]
18141    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
18142        crate::run::pin_test_home();
18143        let home = crate::run::home();
18144
18145        let tmp = tempfile::tempdir().expect("tempdir");
18146        let repo = tmp.path().join("repo");
18147        std::fs::create_dir_all(&repo).unwrap();
18148        init_repo(&repo);
18149        // Unique to this test, so holding its lease cannot collide with
18150        // another test sharing the same process-wide `home`.
18151        let cache_dir = tmp.path().join("target");
18152
18153        let mut config = Config::default();
18154        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
18155        // `verify.gate` stays empty (the default). Any accidental cache
18156        // acquisition must report contention immediately, regardless of how
18157        // long unrelated Git subprocesses take under parallel test load.
18158        config.graph.timeout_verify = Some(2);
18159        config.graph.timeout_resource_wait = 0;
18160
18161        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18162        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18163            .expect("no io error acquiring directly")
18164        {
18165            crate::cache::AcquireOutcome::Acquired(g) => g,
18166            crate::cache::AcquireOutcome::Busy(b) => {
18167                panic!("expected the direct acquire to win the lease first: {b:?}")
18168            }
18169        };
18170
18171        let mut state = RunState::new(
18172            repo.clone(),
18173            "main".to_owned(),
18174            "deadbeef".to_owned(),
18175            "task".to_owned(),
18176            config,
18177        );
18178        state.candidates = vec![Candidate {
18179            index: 0,
18180            label: 'A',
18181            agent: "alpha".to_owned(),
18182            branch: "does-not-exist".to_owned(),
18183            worktree: repo.clone(),
18184            summary: String::new(),
18185            stat: String::new(),
18186            files: 0,
18187            commits: 0,
18188            empty: false,
18189            failed: None,
18190            verified_noop: None,
18191            duration_ms: 0,
18192            folded: false,
18193        }];
18194        state.tally = Some(Tally {
18195            first_choice: BTreeMap::from([('A', 1)]),
18196            borda: BTreeMap::new(),
18197            winner: 'A',
18198            rankings: 1,
18199            unanimous_initial: true,
18200            deliberated: false,
18201            changed_votes: 0,
18202            unanimous_final: true,
18203            tie_break: None,
18204            judges: 0,
18205            present: 0,
18206            quorum: 0,
18207            met_quorum: true,
18208            uncontested: Some("only candidate A produced a change".to_owned()),
18209        });
18210        state.reviews = vec![ReviewRound {
18211            round: 1,
18212            head: "deadbeef".to_owned(),
18213            verified_head: None,
18214            verified_at: None,
18215            reviews: Vec::new(),
18216            e2e: Vec::new(),
18217            fix: None,
18218            blocking: 0,
18219            answered: 0,
18220            expected: 0,
18221            clean: true,
18222            verify_retried: false,
18223            e2e_deferred: false,
18224            e2e_defer_reason: None,
18225            progressed: false,
18226            vote_split: false,
18227            reconsideration: Vec::new(),
18228            verdict: None,
18229        }];
18230
18231        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
18232        state.reviews[0].e2e = vec![CommandOutcome {
18233            command: state.config.verify.e2e[0].clone(),
18234            code: Some(0),
18235            output_tail: String::new(),
18236            duration_ms: 0,
18237            resource_blocked: false,
18238        }];
18239        let mut runner = Runner {
18240            state,
18241            roles: ResolvedRoles {
18242                implementers: Vec::new(),
18243                judges: Vec::new(),
18244                reviewers: Vec::new(),
18245                fixer: None,
18246                conductor: conductor(),
18247                implementer_roster: Vec::new(),
18248                judge_roster: Vec::new(),
18249                reviewer_roster: Vec::new(),
18250            },
18251            sem: Arc::new(Semaphore::new(1)),
18252            pause: Pause::new(),
18253            interrupt: Pause::new(),
18254        };
18255
18256        runner.gate().await.expect("gate");
18257        assert!(runner.state.resource_block.is_none());
18258        assert!(
18259            !runner
18260                .state
18261                .events
18262                .iter()
18263                .any(|event| { event.message.contains("waiting for the shared build cache") }),
18264            "a gate with nothing to run must never request the held cache lease"
18265        );
18266        assert!(
18267            runner.state.gate_ran,
18268            "zero commands is still a real, immediate attempt"
18269        );
18270        assert!(runner.state.gate.is_empty());
18271        assert_ne!(
18272            runner.state.status,
18273            RunStatus::Blocked,
18274            "must not read as resource-blocked on a lease it never asked for"
18275        );
18276    }
18277
18278    /// A separate process carries a live worker's environment throughout all
18279    /// verification entry points; the parallel harness never changes its env.
18280    #[cfg(unix)]
18281    #[tokio::test]
18282    async fn verification_is_standalone_inside_a_live_worker() {
18283        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
18284        if std::env::var_os(CHILD).is_none() {
18285            let home = tempfile::tempdir().unwrap();
18286            let generation = crate::worker::pin_current(home.path()).unwrap();
18287            let output = std::process::Command::new(std::env::current_exe().unwrap())
18288                .args([
18289                    "--exact",
18290                    "graph::tests::verification_is_standalone_inside_a_live_worker",
18291                    "--nocapture",
18292                ])
18293                .env(CHILD, "1")
18294                .env("MAGI_HOME", home.path())
18295                .env("MAGI_WORKER_HOME", home.path())
18296                .env("MAGI_WORKER_ID", "abcdef")
18297                .env("MAGI_WORKER_GENERATION", &generation.id)
18298                .env("MAGI_WORKER_EXE", &generation.executable)
18299                .env("MAGI_RUN", "verification-parent")
18300                .env("MAGI_NODE", "gate")
18301                .env("MAGI_SEAT", "worker-fixture")
18302                .env("MAGI_PROCESS_OWNER", "fixture-owner")
18303                .env("MAGI_NO_AUTOUPDATE", "0")
18304                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
18305                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
18306                .env(
18307                    "VERIFY_EXPECT_TOOLCHAIN",
18308                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
18309                )
18310                .env(
18311                    "VERIFY_EXPECT_CACHE",
18312                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
18313                )
18314                .output()
18315                .unwrap();
18316            assert!(
18317                output.status.success()
18318                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
18319                "{}\n{}",
18320                String::from_utf8_lossy(&output.stdout),
18321                String::from_utf8_lossy(&output.stderr)
18322            );
18323            return;
18324        }
18325        let home = crate::run::home();
18326        let directory = home.join("workers/jobs/abcdef");
18327        std::fs::create_dir_all(&directory).unwrap();
18328        let job = crate::worker::Job {
18329            id: "abcdef".into(),
18330            sequence: 1,
18331            kind: "task".into(),
18332            key: "task:verification".into(),
18333            payload: serde_json::json!({}),
18334            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
18335            version: Some(env!("CARGO_PKG_VERSION").into()),
18336            status: crate::worker::Status::Running,
18337            result: None,
18338            pid: Some(std::process::id()),
18339            created_at: 0,
18340            finished_at: None,
18341        };
18342        std::fs::write(
18343            directory.join("job.json"),
18344            serde_json::to_vec(&job).unwrap(),
18345        )
18346        .unwrap();
18347        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
18348        let assert_worker = || {
18349            assert_eq!(
18350                crate::worker::current_job(&home).unwrap().unwrap().id,
18351                job.id
18352            );
18353            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
18354            assert_eq!(
18355                crate::worker::helper_executable(&home).unwrap(),
18356                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
18357            );
18358            assert!(
18359                crate::updater::Checker::new(&crate::config::Update {
18360                    mode: crate::config::UpdateMode::Notify,
18361                    interval: None,
18362                    auto_restart: false,
18363                })
18364                .is_none(),
18365                "the driver remains pinned, even while tests are standalone"
18366            );
18367        };
18368        assert_worker();
18369        let repo = home.join("repo");
18370        std::fs::create_dir_all(&repo).unwrap();
18371        init_repo(&repo);
18372        // Keep storage, seat/control context, containment and tool lookup.
18373        // No inherited values (or credentials) are printed on success.
18374        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
18375            test -z "${MAGI_WORKER_GENERATION+x}" &&
18376            test -z "${MAGI_WORKER_HOME+x}" &&
18377            test -z "${MAGI_WORKER_EXE+x}" &&
18378            test "$MAGI_HOME/repo" -ef . &&
18379            test "$MAGI_RUN" = verification-parent &&
18380            test "$MAGI_NODE" = gate &&
18381            test "$MAGI_SEAT" = worker-fixture &&
18382            test -n "$MAGI_PROCESS_OWNER" &&
18383            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
18384            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
18385            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
18386            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
18387        // Negative control: an ordinary child inherits the contamination.
18388        assert!(
18389            !std::process::Command::new("sh")
18390                .args(["-c", probe])
18391                .current_dir(&repo)
18392                .status()
18393                .unwrap()
18394                .success()
18395        );
18396        let unrelated_home = tempfile::tempdir().unwrap();
18397        assert!(
18398            crate::worker::current_job(unrelated_home.path()).is_err(),
18399            "an inherited worker id cannot be resolved in a unit fixture's home"
18400        );
18401        let mut runner = runner_at(RunStatus::Gating);
18402        runner.state.repo = repo.clone();
18403        runner.state.config.disk.min_free_bytes = 0;
18404        runner.state.config.coordination = crate::config::test_coordination();
18405        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
18406        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
18407        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
18408        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
18409        let winner = Candidate {
18410            index: 0,
18411            label: 'A',
18412            agent: "fixture".into(),
18413            branch: "main".into(),
18414            worktree: repo.clone(),
18415            summary: String::new(),
18416            stat: String::new(),
18417            files: 0,
18418            commits: 0,
18419            empty: false,
18420            failed: None,
18421            verified_noop: None,
18422            duration_ms: 0,
18423            folded: false,
18424        };
18425        let (e2e, retried, pids) = run_e2e_with_retry(
18426            &mut runner.state,
18427            &["sh".into(), "-c".into()],
18428            &[format!("{probe} && printf initial")],
18429            &repo,
18430            Duration::from_secs(10),
18431            "initial",
18432        )
18433        .await;
18434        assert!(!retried && pids.is_empty());
18435        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
18436        assert_eq!(e2e[0].output_tail, "initial");
18437        let gate = runner.run_gate(&winner).await.unwrap();
18438        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
18439        // An interrupted repair must recheck e2e from saved state. Landing
18440        // reuses that state and must run both pre-gate and gate again.
18441        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
18442            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
18443            passed: false,
18444            outcomes: Vec::new(),
18445        });
18446        runner.state.save().unwrap();
18447        let mut resumed = RunState::load(&runner.state.id).unwrap();
18448        let wait = crate::repo_lock::WaitPolicy::default();
18449        assert!(matches!(
18450            check_gate_repair_e2e(&mut resumed, &winner, &wait)
18451                .await
18452                .unwrap(),
18453            GateFix::Retry
18454        ));
18455        let saved = RunState::load(&resumed.id).unwrap();
18456        let repair = saved.gate_repair_e2e.unwrap();
18457        assert!(repair.passed);
18458        assert_eq!(repair.outcomes.len(), 1);
18459        assert!(repair.outcomes[0].ok());
18460        let landing = run_landing_gate(&mut resumed, &winner, &wait)
18461            .await
18462            .unwrap();
18463        assert_eq!(landing.len(), 1);
18464        assert!(landing[0].ok(), "{landing:?}");
18465        assert_eq!(
18466            std::fs::read_to_string(repo.join("phases")).unwrap(),
18467            "pre\ngate\ne2e\npre\ngate\n"
18468        );
18469        assert_worker();
18470    }
18471
18472    #[cfg(unix)]
18473    #[tokio::test]
18474    async fn verification_keeps_the_panic_and_the_full_output() {
18475        crate::run::pin_test_home();
18476        let tmp = tempfile::tempdir().unwrap();
18477        let mut config = Config::default();
18478        config.disk.min_free_bytes = 0;
18479        let mut state = RunState::new(
18480            tmp.path().to_owned(),
18481            "main".into(),
18482            "base".into(),
18483            "verification evidence".into(),
18484            config,
18485        );
18486        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
18487                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
18488                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
18489        let mut names = Vec::new();
18490        for _ in 0..2 {
18491            let (outcomes, _) = run_commands(
18492                &mut state,
18493                "gate",
18494                "gate",
18495                0,
18496                &["sh".into(), "-c".into()],
18497                &[script.into()],
18498                tmp.path(),
18499                Duration::from_secs(60),
18500            )
18501            .await;
18502            let tail = &outcomes[0].output_tail;
18503            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
18504            assert!(tail.contains("boom assertion"));
18505            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
18506            let line = tail.lines().next().unwrap();
18507            names.push(line.to_owned());
18508        }
18509        assert_ne!(names[0], names[1], "the second run must not overwrite");
18510        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
18511        assert!(full.len() > 500_000 && full.contains("boom assertion"));
18512        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
18513
18514        // Storage failures are evidence too; preserve the command's verdict.
18515        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
18516        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
18517        let (outcomes, _) = run_commands(
18518            &mut state,
18519            "gate",
18520            "gate",
18521            0,
18522            &["sh".into(), "-c".into()],
18523            &["echo failure-output; exit 7".into()],
18524            tmp.path(),
18525            Duration::from_secs(10),
18526        )
18527        .await;
18528        assert_eq!(outcomes[0].code, Some(7));
18529        assert!(state.events.iter().any(|e| {
18530            e.node == "gate" && e.message.contains("could not save the full gate output:")
18531        }));
18532    }
18533
18534    #[cfg(unix)]
18535    #[tokio::test]
18536    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
18537        crate::run::pin_test_home();
18538        let tmp = tempfile::tempdir().unwrap();
18539        let mut config = Config::default();
18540        config.disk.min_free_bytes = 0;
18541        let mut state = RunState::new(
18542            tmp.path().to_owned(),
18543            "main".into(),
18544            "base".into(),
18545            "verification evidence".into(),
18546            config,
18547        );
18548        let (outcomes, pids) = run_commands(
18549            &mut state,
18550            "gate",
18551            "gate",
18552            1,
18553            &["sh".into(), "-c".into()],
18554            &[
18555                "printf before-timeout; exec sleep 30".into(),
18556                "kill -TERM $$".into(),
18557                "exit 7".into(),
18558                "printf success".into(),
18559            ],
18560            tmp.path(),
18561            Duration::from_secs(1),
18562        )
18563        .await;
18564        wait_for_timed_out_children_to_die(&pids).await;
18565        assert_eq!(
18566            pids.len(),
18567            1,
18568            "unleased timeout retains its death-check PID"
18569        );
18570        assert_eq!(outcomes.len(), 4);
18571        assert_eq!(outcomes[0].code, None);
18572        assert!(outcomes[0].output_tail.contains("before-timeout"));
18573        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
18574        assert_eq!(outcomes[1].code, None);
18575        assert!(outcomes[1].output_tail.contains("command terminated:"));
18576        assert!(!outcomes[1].output_tail.contains("timed out"));
18577        assert_eq!(outcomes[2].code, Some(7));
18578        assert!(!outcomes[2].output_tail.contains("timed out"));
18579        assert_eq!(outcomes[3].code, Some(0));
18580        assert_eq!(outcomes[3].output_tail, "success");
18581        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
18582    }
18583
18584    /// The addendum's second gap: a `verify.gate` command running for real
18585    /// wall-clock time had nothing at all to show for it in `active` before
18586    /// `run_commands` learned to record it — a run could sit in `Gating` for
18587    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
18588    /// what was actually happening. Proven with a genuinely still-running
18589    /// command, not just a before/after check on the final state: a poller
18590    /// task reads the same `run.json` `gate()` is writing, the same way the
18591    /// phone or `magi show` would, while the shell command is still blocked
18592    /// on its own release marker.
18593    #[tokio::test]
18594    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
18595        crate::run::pin_test_home();
18596
18597        let tmp = tempfile::tempdir().expect("tempdir");
18598        let repo = tmp.path().join("repo");
18599        std::fs::create_dir_all(&repo).unwrap();
18600        init_repo(&repo);
18601
18602        let mut config = Config::default();
18603        config.verify.gate = vec![
18604            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
18605             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
18606                .to_owned(),
18607        ];
18608
18609        let mut state = RunState::new(
18610            repo.clone(),
18611            "main".to_owned(),
18612            "deadbeef".to_owned(),
18613            "task".to_owned(),
18614            config,
18615        );
18616        let run_id = state.id.clone();
18617        state.candidates = vec![Candidate {
18618            index: 0,
18619            label: 'A',
18620            agent: "alpha".to_owned(),
18621            branch: "does-not-exist".to_owned(),
18622            worktree: repo.clone(),
18623            summary: String::new(),
18624            stat: String::new(),
18625            files: 0,
18626            commits: 0,
18627            empty: false,
18628            failed: None,
18629            verified_noop: None,
18630            duration_ms: 0,
18631            folded: false,
18632        }];
18633        state.tally = Some(Tally {
18634            first_choice: BTreeMap::from([('A', 1)]),
18635            borda: BTreeMap::new(),
18636            winner: 'A',
18637            rankings: 1,
18638            unanimous_initial: true,
18639            deliberated: false,
18640            changed_votes: 0,
18641            unanimous_final: true,
18642            tie_break: None,
18643            judges: 0,
18644            present: 0,
18645            quorum: 0,
18646            met_quorum: true,
18647            uncontested: Some("only candidate A produced a change".to_owned()),
18648        });
18649        state.reviews = vec![ReviewRound {
18650            round: 1,
18651            head: "deadbeef".to_owned(),
18652            verified_head: None,
18653            verified_at: None,
18654            reviews: Vec::new(),
18655            e2e: Vec::new(),
18656            fix: None,
18657            blocking: 0,
18658            answered: 0,
18659            expected: 0,
18660            clean: true,
18661            verify_retried: false,
18662            e2e_deferred: false,
18663            e2e_defer_reason: None,
18664            progressed: false,
18665            vote_split: false,
18666            reconsideration: Vec::new(),
18667            verdict: None,
18668        }];
18669
18670        let mut runner = Runner {
18671            state,
18672            roles: ResolvedRoles {
18673                implementers: Vec::new(),
18674                judges: Vec::new(),
18675                reviewers: Vec::new(),
18676                fixer: None,
18677                conductor: conductor(),
18678                implementer_roster: Vec::new(),
18679                judge_roster: Vec::new(),
18680                reviewer_roster: Vec::new(),
18681            },
18682            sem: Arc::new(Semaphore::new(1)),
18683            pause: Pause::new(),
18684            interrupt: Pause::new(),
18685        };
18686
18687        let started_marker = repo.join("started.marker");
18688        let release_marker = repo.join("release.marker");
18689        let poller = tokio::spawn(async move {
18690            // Bounded so a regression that never records the task entry
18691            // fails this test in seconds instead of hanging the suite —
18692            // the same shape `a_park_requested_while_a_seat_is_mid_call_
18693            // does_not_cut_it_short` uses for the same reason.
18694            for _ in 0..100 {
18695                if started_marker.exists()
18696                    && let Ok(s) = crate::run::RunState::load(&run_id)
18697                    && let Some(a) = s.active.get("gate")
18698                {
18699                    std::fs::write(&release_marker, b"go").expect("release marker");
18700                    return Some(a.clone());
18701                }
18702                tokio::time::sleep(Duration::from_millis(50)).await;
18703            }
18704            None
18705        });
18706
18707        runner.gate().await.expect("gate");
18708        let captured = poller.await.expect("poller task");
18709        let captured = captured.expect(
18710            "the poller never saw a `gate` task entry in run.json while the command was \
18711             still blocked on its own release marker",
18712        );
18713
18714        assert_eq!(captured.task.as_deref(), Some("gate"));
18715        assert_eq!(captured.node, "gate");
18716        assert_eq!(captured.index, Some(1));
18717        assert_eq!(captured.total, Some(1));
18718        assert!(
18719            captured
18720                .command
18721                .as_deref()
18722                .is_some_and(|c| c.contains("started.marker")),
18723            "{captured:?}"
18724        );
18725
18726        assert!(
18727            runner.state.active.is_empty(),
18728            "the entry must be cleared once the command actually finished: {:?}",
18729            runner.state.active
18730        );
18731        assert!(runner.state.gate_ran);
18732        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18733    }
18734
18735    /// The hand-off over a blocking finding a reviewer rejected on leaves a
18736    /// record for `land`; one with only a Minor, or no reject, leaves none.
18737    #[tokio::test]
18738    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
18739        use crate::verdict::{Finding, ReviewVote, Severity};
18740        crate::run::pin_test_home();
18741        let tmp = tempfile::tempdir().expect("tempdir");
18742        let repo = tmp.path().join("repo");
18743        std::fs::create_dir_all(&repo).unwrap();
18744        init_repo(&repo);
18745
18746        for (severity, vote, expect) in [
18747            (Severity::Major, ReviewVote::Reject, true),
18748            (Severity::Minor, ReviewVote::Reject, false),
18749            (Severity::Major, ReviewVote::Approve, false),
18750        ] {
18751            let mut round = review_round(false, 1, 1, 1, false, true);
18752            round.reviews = vec![ReviewRecord {
18753                reviewer: 1,
18754                agent: "alpha".to_owned(),
18755                summary: String::new(),
18756                findings: vec![Finding {
18757                    id: "R1-1-1".to_owned(),
18758                    severity,
18759                    file: None,
18760                    line: None,
18761                    title: "t".to_owned(),
18762                    detail: String::new(),
18763                }],
18764                vote: Some(vote),
18765                failed: None,
18766                duration_ms: 0,
18767                attempts: 0,
18768            }];
18769            let mut state = RunState::new(
18770                repo.clone(),
18771                "main".to_owned(),
18772                "deadbeef".to_owned(),
18773                "task".to_owned(),
18774                Config::default(),
18775            );
18776            state.reviews = vec![round];
18777            let mut runner = Runner {
18778                state,
18779                roles: ResolvedRoles {
18780                    implementers: Vec::new(),
18781                    judges: Vec::new(),
18782                    reviewers: Vec::new(),
18783                    fixer: None,
18784                    conductor: conductor(),
18785                    implementer_roster: Vec::new(),
18786                    judge_roster: Vec::new(),
18787                    reviewer_roster: Vec::new(),
18788                },
18789                sem: Arc::new(Semaphore::new(1)),
18790                pause: Pause::new(),
18791                interrupt: Pause::new(),
18792            };
18793            let shell = runner.state.config.shell();
18794            runner
18795                .stop_reviewing("round budget spent", &shell, &repo)
18796                .await
18797                .expect("stop_reviewing");
18798            assert_eq!(runner.state.status, RunStatus::Gating);
18799            assert_eq!(
18800                runner.state.contested_handoff.is_some(),
18801                expect,
18802                "{severity:?} + {vote:?}"
18803            );
18804        }
18805    }
18806
18807    /// The shape the incident this whole fix responds to actually had: the
18808    /// round budget spent, the last round's own e2e blocked on the shared
18809    /// build cache (held here by a live pid — this test process — exactly
18810    /// `cache`'s own unit tests' pattern for "another owner, still alive"
18811    /// without forking a process). `stop_reviewing` must retry it — not
18812    /// silently leave the round looking untouched (the catch-up-only half of
18813    /// the bug), and not read the contention as a red `e2e` and block the
18814    /// run on it (the other half). Called directly, the same way
18815    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
18816    /// above exercises `gate`, so this never needs a real cargo build to
18817    /// reach: the lease is never released, so `with_cache_lease` never gets
18818    /// past acquiring it into anything that would need a real workspace.
18819    #[tokio::test]
18820    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
18821        crate::run::pin_test_home();
18822        let home = crate::run::home();
18823
18824        let tmp = tempfile::tempdir().expect("tempdir");
18825        let repo = tmp.path().join("repo");
18826        std::fs::create_dir_all(&repo).unwrap();
18827        init_repo(&repo);
18828        let head = crate::git::rev_parse(&repo, "HEAD")
18829            .await
18830            .expect("rev-parse");
18831        // Unique to this test, so holding its lease cannot collide with
18832        // another test sharing the same process-wide `home`.
18833        let cache_dir = tmp.path().join("target");
18834
18835        let mut config = Config::default();
18836        config.verify.e2e = vec![format!(
18837            "CARGO_TARGET_DIR='{}' test -f README.md",
18838            cache_dir.display()
18839        )];
18840        config.graph.review_rounds = 1;
18841        // Bounded so a regression that does start waiting fails the test in
18842        // seconds, not hangs it.
18843        config.graph.timeout_verify = Some(2);
18844        config.graph.timeout_resource_wait = 2;
18845
18846        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18847        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18848            .expect("no io error acquiring directly")
18849        {
18850            crate::cache::AcquireOutcome::Acquired(g) => g,
18851            crate::cache::AcquireOutcome::Busy(b) => {
18852                panic!("expected the direct acquire to win the lease first: {b:?}")
18853            }
18854        };
18855
18856        let mut state = RunState::new(
18857            repo.clone(),
18858            "main".to_owned(),
18859            head.clone(),
18860            "task".to_owned(),
18861            config,
18862        );
18863        state.candidates = vec![Candidate {
18864            index: 0,
18865            label: 'A',
18866            agent: "alpha".to_owned(),
18867            branch: "does-not-exist".to_owned(),
18868            worktree: repo.clone(),
18869            summary: String::new(),
18870            stat: String::new(),
18871            files: 0,
18872            commits: 0,
18873            empty: false,
18874            failed: None,
18875            verified_noop: None,
18876            duration_ms: 0,
18877            folded: false,
18878        }];
18879        state.tally = Some(Tally {
18880            first_choice: BTreeMap::from([('A', 1)]),
18881            borda: BTreeMap::new(),
18882            winner: 'A',
18883            rankings: 1,
18884            unanimous_initial: true,
18885            deliberated: false,
18886            changed_votes: 0,
18887            unanimous_final: true,
18888            tie_break: None,
18889            judges: 0,
18890            present: 0,
18891            quorum: 0,
18892            met_quorum: true,
18893            uncontested: Some("only candidate A produced a change".to_owned()),
18894        });
18895        // The round budget's last round, deferred: `needs_catchup_run`'s
18896        // other trigger. `stop_reviewing`'s retry machinery must treat this
18897        // exactly like a resource-blocked attempt once it actually runs.
18898        state.reviews = vec![ReviewRound {
18899            round: 1,
18900            head: head.clone(),
18901            verified_head: None,
18902            verified_at: None,
18903            reviews: Vec::new(),
18904            e2e: Vec::new(),
18905            fix: None,
18906            blocking: 1,
18907            answered: 1,
18908            expected: 1,
18909            clean: false,
18910            verify_retried: false,
18911            e2e_deferred: true,
18912            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
18913            progressed: false,
18914            vote_split: false,
18915            reconsideration: Vec::new(),
18916            verdict: None,
18917        }];
18918
18919        let mut runner = Runner {
18920            state,
18921            roles: ResolvedRoles {
18922                implementers: Vec::new(),
18923                judges: Vec::new(),
18924                reviewers: Vec::new(),
18925                fixer: None,
18926                conductor: conductor(),
18927                implementer_roster: Vec::new(),
18928                judge_roster: Vec::new(),
18929                reviewer_roster: Vec::new(),
18930            },
18931            sem: Arc::new(Semaphore::new(1)),
18932            pause: Pause::new(),
18933            interrupt: Pause::new(),
18934        };
18935
18936        let shell = runner.state.config.shell();
18937        runner
18938            .stop_reviewing("round budget spent", &shell, &repo)
18939            .await
18940            .expect("stop_reviewing");
18941
18942        let last = runner.state.reviews.last().expect("round record");
18943        assert_eq!(
18944            last.e2e_status(),
18945            E2eStatus::ResourceBlocked,
18946            "the shared cache is still held; the attempt must read as blocked, not deferred or \
18947             failed: {last:?}"
18948        );
18949        assert_eq!(
18950            last.verified_head.as_deref(),
18951            Some(head.as_str()),
18952            "which commit this attempt targeted is known even though nothing finished checking \
18953             it"
18954        );
18955        let first_attempt_at = last
18956            .verified_at
18957            .expect("when this attempt ran is known too");
18958        assert_ne!(
18959            runner.state.status,
18960            RunStatus::Blocked,
18961            "contention is evidence about the machine, not the patch — it must not settle the \
18962             run as blocked: {:?}",
18963            runner.state.status
18964        );
18965        assert!(
18966            !runner
18967                .state
18968                .events
18969                .iter()
18970                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
18971            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
18972            runner.state.events
18973        );
18974
18975        // The cache is still held: a later reentry must retry the same
18976        // round's verification again — not leave it looking exactly as
18977        // untouched as the first blocked attempt, which is indistinguishable
18978        // from never having tried again at all.
18979        runner
18980            .stop_reviewing("round budget spent", &shell, &repo)
18981            .await
18982            .expect("stop_reviewing retry");
18983        assert_eq!(
18984            runner.state.reviews.len(),
18985            1,
18986            "no new round was started: {:?}",
18987            runner.state.reviews
18988        );
18989        let last = runner.state.reviews.last().expect("round record");
18990        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
18991        assert!(
18992            last.verified_at.expect("still known") > first_attempt_at,
18993            "a second reentry must be a fresh attempt, not a stale copy of the first"
18994        );
18995        assert_ne!(runner.state.status, RunStatus::Blocked);
18996
18997        held.release();
18998    }
18999
19000    #[cfg(unix)]
19001    #[tokio::test]
19002    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
19003        ask_test_home();
19004        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
19005            let tmp = tempfile::tempdir().unwrap();
19006            let repo = tmp.path().join("repo");
19007            std::fs::create_dir(&repo).unwrap();
19008            init_repo(&repo);
19009            let run_git = |args: &[&str]| {
19010                let out = std::process::Command::new("git")
19011                    .args(args)
19012                    .current_dir(&repo)
19013                    .output()
19014                    .unwrap();
19015                assert!(
19016                    out.status.success(),
19017                    "{args:?}: {}",
19018                    String::from_utf8_lossy(&out.stderr)
19019                );
19020            };
19021            let origin = tmp.path().join("origin.git");
19022            run_git(&["init", "--bare", origin.to_str().unwrap()]);
19023            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
19024            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
19025            run_git(&["add", "spec.rs"]);
19026            run_git(&["commit", "-m", "initial spec"]);
19027            run_git(&["push", "origin", "main"]);
19028            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
19029            run_git(&["checkout", "-b", "fix/review"]);
19030            std::fs::write(
19031                repo.join("check.rs"),
19032                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
19033            )
19034            .unwrap();
19035            run_git(&["add", "check.rs"]);
19036            run_git(&["commit", "-m", "use spec"]);
19037            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
19038            run_git(&["checkout", "main"]);
19039            std::fs::write(
19040                repo.join("spec.rs"),
19041                "struct AgentSpec { max_turn_secs: u64 }\n",
19042            )
19043            .unwrap();
19044            run_git(&["add", "spec.rs"]);
19045            run_git(&["commit", "-m", "extend spec"]);
19046            run_git(&["push", "origin", "main"]);
19047            run_git(&["checkout", "fix/review"]);
19048
19049            let mut runner = runner_at(RunStatus::Reviewing);
19050            runner.state.repo = repo.clone();
19051            runner.state.base_commit = base;
19052            runner.state.pr_head = Some("fix/review".into());
19053            runner.state.config.graph.review_rounds = max_rounds;
19054            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
19055            runner.state.config.graph.sessions = false;
19056            runner.state.config.disk.min_free_bytes = 0;
19057            runner.state.config.verify.e2e =
19058                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
19059            runner.state.candidates = vec![Candidate {
19060                index: 0,
19061                label: 'A',
19062                agent: "fixture".into(),
19063                branch: "fix/review".into(),
19064                worktree: repo.clone(),
19065                summary: String::new(),
19066                stat: String::new(),
19067                files: 1,
19068                commits: 1,
19069                empty: false,
19070                failed: None,
19071                verified_noop: None,
19072                duration_ms: 0,
19073                folded: false,
19074            }];
19075            runner.state.tally = Some(Tally {
19076                first_choice: BTreeMap::new(),
19077                borda: BTreeMap::new(),
19078                winner: 'A',
19079                rankings: 0,
19080                unanimous_initial: true,
19081                deliberated: false,
19082                changed_votes: 0,
19083                unanimous_final: true,
19084                tie_break: None,
19085                judges: 0,
19086                present: 0,
19087                quorum: 0,
19088                met_quorum: true,
19089                uncontested: Some("review-only".into()),
19090            });
19091            let mut last = review_round(false, 0, 1, 1, false, false);
19092            last.head = reviewed.clone();
19093            last.e2e_deferred = true;
19094            last.e2e[0].resource_blocked = true;
19095            last.e2e[0].output_tail = "waiting for the shared build cache".into();
19096            if max_rounds == 1 {
19097                last.e2e.clear();
19098            }
19099            runner.state.reviews = vec![last];
19100            let calls = tmp.path().join("fix-calls");
19101            let mut fixer = spec("repair");
19102            fixer
19103                .env
19104                .insert("FIX_CALLS".into(), calls.display().to_string());
19105            fixer.command = vec!["sh".into(), "-c".into(), r#"
19106                echo fix >> "$FIX_CALLS"
19107                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
19108                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
19109                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
19110                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
19111            "#.into()];
19112            let mut reviewer = spec("reviewer");
19113            reviewer.command = vec![
19114                "sh".into(),
19115                "-c".into(),
19116                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
19117                    .into(),
19118            ];
19119            runner.roles.fixer = Some(vec![fixer]);
19120            runner.roles.reviewers = vec![reviewer];
19121
19122            runner.sync_to_base().await.unwrap();
19123            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
19124            assert_ne!(rebased, reviewed);
19125            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
19126            if reentry {
19127                // Simulate a crash after catch-up evidence was saved but before
19128                // dispatching the fixer. No dedicated pending-repair field exists.
19129                let shell = runner.state.config.shell();
19130                runner
19131                    .stop_unresolved_review("deferred verification", &shell, &repo)
19132                    .await
19133                    .unwrap();
19134                assert!(runner.catchup_repairable());
19135                assert!(!calls.exists());
19136                runner.state = RunState::load(&runner.state.id).unwrap();
19137                assert!(runner.catchup_repairable());
19138            }
19139            runner.review_loop().await.unwrap();
19140            let first = &runner.state.reviews[0];
19141            assert_eq!(first.head, reviewed);
19142            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
19143            assert!(
19144                first.verify_retried,
19145                "a reproduced build failure gets exactly one retry: {:?}",
19146                first.e2e
19147            );
19148            assert!(first.e2e[0].output_tail.contains("E0063"));
19149            assert!(!runner.catchup_repairable());
19150            if max_rounds == 2 {
19151                assert_eq!(
19152                    runner.state.status,
19153                    RunStatus::Gating,
19154                    "{}",
19155                    crate::report::run(&runner.state)
19156                );
19157                assert_eq!(runner.state.reviews.len(), 2);
19158                assert!(first.fix.as_ref().unwrap().committed);
19159                assert!(runner.state.reviews[1].clean);
19160                assert_eq!(
19161                    runner
19162                        .state
19163                        .reviews
19164                        .iter()
19165                        .filter(|r| r.fix.is_some())
19166                        .count(),
19167                    1
19168                );
19169                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
19170                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
19171            } else {
19172                assert_eq!(runner.state.status, RunStatus::Blocked);
19173                assert!(first.fix.is_none());
19174                assert!(!calls.exists());
19175                let recovery = crate::recovery::assess(
19176                    "prior",
19177                    Some(&runner.state),
19178                    &runner.landing_base(),
19179                    crate::recovery::Relation::Diverged,
19180                );
19181                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
19182                let reasons = recovery.reasons.join("\n");
19183                assert!(
19184                    reasons.contains(&reviewed) && reasons.contains(&rebased),
19185                    "{reasons}"
19186                );
19187                assert!(
19188                    reasons.contains("base sync to")
19189                        && reasons.contains("additional review is required"),
19190                    "{reasons}"
19191                );
19192            }
19193        }
19194    }
19195
19196    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
19197    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
19198    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
19199    /// own top-of-function fast path (`review_conclusion`) correctly reads
19200    /// this shape as `None` rather than guessing `Blocked`, and the loop's
19201    /// own `for` range is empty once the round budget is spent, so
19202    /// `review_loop` must retry the check itself rather than silently doing
19203    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
19204    /// above exercises directly, but through `review_loop`'s own entry point
19205    /// this time, proving the wiring between the two rather than just the
19206    /// retry logic in isolation.
19207    #[tokio::test]
19208    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
19209        crate::run::pin_test_home();
19210        let home = crate::run::home();
19211
19212        let tmp = tempfile::tempdir().expect("tempdir");
19213        let repo = tmp.path().join("repo");
19214        std::fs::create_dir_all(&repo).unwrap();
19215        init_repo(&repo);
19216        let head = crate::git::rev_parse(&repo, "HEAD")
19217            .await
19218            .expect("rev-parse");
19219        let cache_dir = tmp.path().join("target");
19220
19221        let mut config = Config::default();
19222        config.verify.e2e = vec![format!(
19223            "CARGO_TARGET_DIR='{}' test -f README.md",
19224            cache_dir.display()
19225        )];
19226        config.graph.review_rounds = 1;
19227        config.graph.timeout_verify = Some(2);
19228        config.graph.timeout_resource_wait = 2;
19229
19230        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19231        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19232            .expect("no io error acquiring directly")
19233        {
19234            crate::cache::AcquireOutcome::Acquired(g) => g,
19235            crate::cache::AcquireOutcome::Busy(b) => {
19236                panic!("expected the direct acquire to win the lease first: {b:?}")
19237            }
19238        };
19239
19240        let mut state = RunState::new(
19241            repo.clone(),
19242            "main".to_owned(),
19243            head.clone(),
19244            "task".to_owned(),
19245            config,
19246        );
19247        state.candidates = vec![Candidate {
19248            index: 0,
19249            label: 'A',
19250            agent: "alpha".to_owned(),
19251            branch: "does-not-exist".to_owned(),
19252            worktree: repo.clone(),
19253            summary: String::new(),
19254            stat: String::new(),
19255            files: 0,
19256            commits: 0,
19257            empty: false,
19258            failed: None,
19259            verified_noop: None,
19260            duration_ms: 0,
19261            folded: false,
19262        }];
19263        state.tally = Some(Tally {
19264            first_choice: BTreeMap::from([('A', 1)]),
19265            borda: BTreeMap::new(),
19266            winner: 'A',
19267            rankings: 1,
19268            unanimous_initial: true,
19269            deliberated: false,
19270            changed_votes: 0,
19271            unanimous_final: true,
19272            tie_break: None,
19273            judges: 0,
19274            present: 0,
19275            quorum: 0,
19276            met_quorum: true,
19277            uncontested: Some("only candidate A produced a change".to_owned()),
19278        });
19279        // The exact shape a prior process's `stop_reviewing` would have left
19280        // on disk: the round budget's last round, a real attempt already
19281        // made and already resource-blocked.
19282        state.reviews = vec![ReviewRound {
19283            round: 1,
19284            head: head.clone(),
19285            verified_head: Some(head.clone()),
19286            verified_at: Some(jiff::Timestamp::now()),
19287            reviews: Vec::new(),
19288            e2e: vec![CommandOutcome {
19289                command: format!(
19290                    "CARGO_TARGET_DIR='{}' test -f README.md",
19291                    cache_dir.display()
19292                ),
19293                code: None,
19294                output_tail: "waiting for the shared build cache".to_owned(),
19295                duration_ms: 0,
19296                resource_blocked: true,
19297            }],
19298            fix: None,
19299            blocking: 1,
19300            answered: 1,
19301            expected: 1,
19302            clean: false,
19303            verify_retried: false,
19304            e2e_deferred: false,
19305            e2e_defer_reason: None,
19306            progressed: false,
19307            vote_split: false,
19308            reconsideration: Vec::new(),
19309            verdict: None,
19310        }];
19311
19312        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
19313        let mut runner = Runner {
19314            state,
19315            roles: ResolvedRoles {
19316                implementers: Vec::new(),
19317                judges: Vec::new(),
19318                reviewers: Vec::new(),
19319                fixer: None,
19320                conductor: conductor(),
19321                implementer_roster: Vec::new(),
19322                judge_roster: Vec::new(),
19323                reviewer_roster: Vec::new(),
19324            },
19325            sem: Arc::new(Semaphore::new(1)),
19326            pause: Pause::new(),
19327            interrupt: Pause::new(),
19328        };
19329
19330        // The lease is still held throughout, so this reentry's own retry is
19331        // also contended — proving `review_loop` actually tried again (not
19332        // that it happened to succeed) is what the timestamp comparison
19333        // below is for.
19334        runner.review_loop().await.expect("review_loop");
19335
19336        assert_eq!(
19337            runner.state.reviews.len(),
19338            1,
19339            "no new round was started on top of the unresolved one: {:?}",
19340            runner.state.reviews
19341        );
19342        let last = &runner.state.reviews[0];
19343        assert_eq!(
19344            last.e2e_status(),
19345            E2eStatus::ResourceBlocked,
19346            "still contended: {last:?}"
19347        );
19348        assert!(
19349            last.verified_at.expect("still known") > first_attempt_at,
19350            "review_loop must have actually retried the check, not left it exactly as found"
19351        );
19352        assert_ne!(
19353            runner.state.status,
19354            RunStatus::Blocked,
19355            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
19356            runner.state.status
19357        );
19358
19359        held.release();
19360    }
19361
19362    #[tokio::test]
19363    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
19364        crate::run::pin_test_home();
19365        for status in [RunStatus::Blocked, RunStatus::Gating] {
19366            let mut runner = runner_at(status);
19367            runner.state.config.graph.land = true;
19368            runner.state.gate_ran = true;
19369            runner.state.merge = Some(MergeOutcome {
19370                mode: MergeMode::Pr,
19371                ok: false,
19372                detail: "the fixer produced no commit".into(),
19373                empty: false,
19374            });
19375            let url = "https://example.invalid/o/r/pull/1";
19376            runner.state.pr = Some(crate::run::PrRecord {
19377                url: url.into(),
19378                number: 1,
19379                state: "open".into(),
19380                checks: "red".into(),
19381                round: 1,
19382                rounds: 1,
19383                red_at_merge: Vec::new(),
19384            });
19385            runner.state.save().unwrap();
19386            // The nonexistent repository makes observation fail before a forge
19387            // command can run. We still must reach land, rather than silently
19388            // return from the graph with its old gate and merge records.
19389            runner.execute_graph().await.unwrap();
19390            assert_eq!(runner.state.status, RunStatus::Blocked);
19391            assert!(runner.state.land_sync_pending);
19392            assert!(runner.state.events.iter().any(|event|
19393                event.node == "land" && event.message == format!("watching {url}")));
19394            assert_eq!(
19395                runner.state.merge.as_ref().unwrap().detail,
19396                "the fixer produced no commit"
19397            );
19398        }
19399    }
19400
19401    #[tokio::test]
19402    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
19403        crate::run::pin_test_home();
19404        let tmp = tempfile::tempdir().expect("tempdir");
19405        let repo = tmp.path().join("repo");
19406        std::fs::create_dir_all(&repo).unwrap();
19407        init_repo(&repo);
19408
19409        let mut config = Config::default();
19410        config.merge.mode = MergeMode::Pr;
19411        config.graph.land = true;
19412        config.graph.land_approval = false;
19413
19414        let mut state = RunState::new(
19415            repo.clone(),
19416            "main".to_owned(),
19417            "deadbeef".to_owned(),
19418            "task".to_owned(),
19419            config,
19420        );
19421        state.candidates = vec![Candidate {
19422            index: 0,
19423            label: 'A',
19424            agent: "alpha".to_owned(),
19425            branch: "does-not-exist".to_owned(),
19426            worktree: repo.clone(),
19427            summary: String::new(),
19428            stat: String::new(),
19429            files: 0,
19430            commits: 0,
19431            empty: false,
19432            failed: None,
19433            verified_noop: None,
19434            duration_ms: 0,
19435            folded: false,
19436        }];
19437        state.tally = Some(Tally {
19438            first_choice: BTreeMap::from([('A', 1)]),
19439            borda: BTreeMap::new(),
19440            winner: 'A',
19441            rankings: 1,
19442            unanimous_initial: true,
19443            deliberated: false,
19444            changed_votes: 0,
19445            unanimous_final: true,
19446            tie_break: None,
19447            judges: 0,
19448            present: 0,
19449            quorum: 0,
19450            met_quorum: true,
19451            uncontested: Some("only candidate A produced a change".to_owned()),
19452        });
19453        state.reviews = vec![ReviewRound {
19454            round: 1,
19455            head: "deadbeef".to_owned(),
19456            verified_head: None,
19457            verified_at: None,
19458            reviews: Vec::new(),
19459            e2e: Vec::new(),
19460            fix: None,
19461            blocking: 0,
19462            answered: 0,
19463            expected: 0,
19464            clean: true,
19465            verify_retried: false,
19466            e2e_deferred: false,
19467            e2e_defer_reason: None,
19468            progressed: false,
19469            vote_split: false,
19470            reconsideration: Vec::new(),
19471            verdict: None,
19472        }];
19473        state.gate = vec![CommandOutcome {
19474            command: "test".to_owned(),
19475            code: Some(0),
19476            output_tail: String::new(),
19477            duration_ms: 0,
19478            resource_blocked: false,
19479        }];
19480        state.gate_ran = true;
19481        // A first pass through `merge` already pushed and opened this pull
19482        // request; `status` is `Landing` because a previous call into `land`
19483        // parked or was interrupted before it reached a terminal outcome.
19484        state.status = RunStatus::Landing;
19485        state.merge = Some(MergeOutcome {
19486            mode: MergeMode::Pr,
19487            ok: true,
19488            detail: "https://example.invalid/x/y/pull/1".to_owned(),
19489            empty: false,
19490        });
19491
19492        // The Landing-resume shortcut calls `run_land` directly rather than
19493        // through `merge`, which is exactly the call site that used to skip
19494        // `settle_questions` - see the fixture below.
19495        ask_test_home();
19496        // A real resume loads a persisted run; driver ownership is checked
19497        // against that record before the Landing shortcut is entered.
19498        state.save().expect("save landing run");
19499        let store = ask::Questions::open();
19500        let q = ask_open_question(&store, &state.id);
19501
19502        let mut runner = Runner {
19503            state,
19504            roles: ResolvedRoles {
19505                implementers: Vec::new(),
19506                judges: Vec::new(),
19507                reviewers: Vec::new(),
19508                fixer: None,
19509                conductor: conductor(),
19510                implementer_roster: Vec::new(),
19511                judge_roster: Vec::new(),
19512                reviewer_roster: Vec::new(),
19513            },
19514            sem: Arc::new(Semaphore::new(1)),
19515            pause: Pause::new(),
19516            interrupt: Pause::new(),
19517        };
19518
19519        // `execute`, not `merge` directly: the Landing-resume shortcut lives
19520        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
19521        // exactly because `review_loop` would otherwise clobber the marker
19522        // first.
19523        runner.execute().await.expect("execute");
19524
19525        assert_eq!(
19526            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19527            Some("https://example.invalid/x/y/pull/1"),
19528            "reentry must not push again or open a second pull request over the \
19529             one `land` is already watching"
19530        );
19531        assert_ne!(
19532            runner.state.status,
19533            RunStatus::Landing,
19534            "land could not actually reach the fake pull request, so it must \
19535             have given up rather than left the run silently parked forever"
19536        );
19537        // `land` could not reach the fake pull request, so it gave up into
19538        // `Blocked` - still resumable, so the question must not have been
19539        // swept just because this branch now also calls `settle_questions`.
19540        assert_eq!(runner.state.status, RunStatus::Blocked);
19541        assert!(
19542            store.get(&q.id).unwrap().status.open(),
19543            "Blocked is still alive; settle_questions must have been a no-op here"
19544        );
19545    }
19546
19547    fn state_with_round(round: ReviewRound) -> RunState {
19548        let mut s = RunState::new(
19549            PathBuf::from("/repo"),
19550            "main".to_owned(),
19551            "abc1234".to_owned(),
19552            "add retries".to_owned(),
19553            Config::default(),
19554        );
19555        s.reviews = vec![round];
19556        s
19557    }
19558
19559    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
19560        crate::verdict::Finding {
19561            id: id.to_owned(),
19562            severity,
19563            file: None,
19564            line: None,
19565            title: title.to_owned(),
19566            detail: String::new(),
19567        }
19568    }
19569
19570    #[test]
19571    fn pr_body_names_open_findings_and_declined_ones() {
19572        let round = ReviewRound {
19573            round: 2,
19574            head: "deadbee".to_owned(),
19575            verified_head: None,
19576            verified_at: None,
19577            reviews: vec![ReviewRecord {
19578                attempts: 0,
19579                reviewer: 1,
19580                agent: "alpha".to_owned(),
19581                summary: String::new(),
19582                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
19583                vote: None,
19584                failed: None,
19585                duration_ms: 0,
19586            }],
19587            e2e: vec![CommandOutcome {
19588                command: "cargo test".to_owned(),
19589                code: Some(0),
19590                output_tail: String::new(),
19591                duration_ms: 0,
19592                resource_blocked: false,
19593            }],
19594            verify_retried: false,
19595            e2e_deferred: false,
19596            e2e_defer_reason: None,
19597            fix: Some(FixRecord {
19598                agent: "alpha".to_owned(),
19599                addressed: Vec::new(),
19600                rejected: vec![crate::verdict::Rejection {
19601                    id: "R1-1-1".to_owned(),
19602                    why: "not reachable from any caller".to_owned(),
19603                }],
19604                notes: String::new(),
19605                committed: true,
19606                failed: None,
19607                duration_ms: 0,
19608                continuation: None,
19609            }),
19610            blocking: 0,
19611            answered: 1,
19612            expected: 1,
19613            clean: false,
19614            progressed: true,
19615            vote_split: false,
19616            reconsideration: Vec::new(),
19617            verdict: None,
19618        };
19619        let state = state_with_round(round);
19620        let body = pr_message(&state, 'A').body;
19621
19622        assert!(body.contains("add retries"), "the task must still be there");
19623        assert!(body.contains("R2-1-1"), "{body}");
19624        assert!(!body.contains("unused import"), "{body}");
19625        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
19626        assert!(
19627            !body.contains("not reachable from any caller"),
19628            "the reason it was declined: {body}"
19629        );
19630    }
19631
19632    #[test]
19633    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
19634        let round = ReviewRound {
19635            round: 1,
19636            head: "deadbee".to_owned(),
19637            verified_head: None,
19638            verified_at: None,
19639            reviews: vec![ReviewRecord {
19640                attempts: 0,
19641                reviewer: 1,
19642                agent: "alpha".to_owned(),
19643                summary: String::new(),
19644                findings: Vec::new(),
19645                vote: None,
19646                failed: None,
19647                duration_ms: 0,
19648            }],
19649            e2e: Vec::new(),
19650            verify_retried: false,
19651            e2e_deferred: false,
19652            e2e_defer_reason: None,
19653            fix: None,
19654            blocking: 0,
19655            answered: 1,
19656            expected: 1,
19657            clean: true,
19658            progressed: false,
19659            vote_split: false,
19660            reconsideration: Vec::new(),
19661            verdict: None,
19662        };
19663        let state = state_with_round(round);
19664        let body = pr_message(&state, 'A').body;
19665        assert!(!body.contains("Open review findings"), "{body}");
19666        assert!(!body.contains("Declined"), "{body}");
19667    }
19668
19669    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
19670        let mut state = RunState::new(
19671            PathBuf::from("/repo"),
19672            "main".to_owned(),
19673            "abc1234".to_owned(),
19674            instruction.to_owned(),
19675            Config::default(),
19676        );
19677        state.candidates.push(Candidate {
19678            index: 0,
19679            label: 'A',
19680            agent: "alpha".to_owned(),
19681            branch: "magi/x/A".to_owned(),
19682            worktree: PathBuf::from("/wt"),
19683            summary: summary.to_owned(),
19684            stat: String::new(),
19685            files: 1,
19686            commits: 1,
19687            empty: false,
19688            failed: None,
19689            verified_noop: None,
19690            folded: false,
19691            duration_ms: 0,
19692        });
19693        state
19694    }
19695
19696    fn review_state(subjects: &[&str]) -> RunState {
19697        let mut state = state_with_summary(
19698            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
19699            "",
19700        );
19701        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19702        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
19703        state
19704    }
19705
19706    /// A branch rebased onto a main that moved past the recorded
19707    /// `base_commit` is titled from its own first commit, never main's.
19708    #[tokio::test]
19709    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
19710        ask_test_home();
19711        let tmp = tempfile::tempdir().unwrap();
19712        let repo = tmp.path().join("repo");
19713        std::fs::create_dir_all(&repo).unwrap();
19714        init_repo(&repo);
19715        let origin = tmp.path().join("origin.git");
19716        let g = |dir: &Path, args: &[&str]| {
19717            let out = std::process::Command::new("git")
19718                .args(args)
19719                .current_dir(dir)
19720                .quiet()
19721                .output()
19722                .expect("spawn git");
19723            assert!(
19724                out.status.success(),
19725                "git {args:?}: {}",
19726                String::from_utf8_lossy(&out.stderr)
19727            );
19728        };
19729        g(
19730            tmp.path(),
19731            &[
19732                "clone",
19733                "--bare",
19734                "-q",
19735                repo.to_str().unwrap(),
19736                origin.to_str().unwrap(),
19737            ],
19738        );
19739        g(
19740            &repo,
19741            &["remote", "add", "origin", origin.to_str().unwrap()],
19742        );
19743        let c1 = git::rev_parse(&repo, "main").await.unwrap();
19744
19745        // Main moves on; the branch is built on top of the new main.
19746        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
19747        g(&repo, &["add", "-A"]);
19748        g(
19749            &repo,
19750            &["commit", "-q", "-m", "chore(deps): update a crate"],
19751        );
19752        g(&repo, &["push", "-q", "origin", "main"]);
19753        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
19754        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19755        g(&repo, &["add", "-A"]);
19756        g(
19757            &repo,
19758            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
19759        );
19760        g(&repo, &["checkout", "-q", "main"]);
19761
19762        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
19763        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
19764        // Without a readable tracking ref the recorded base's merge base is used.
19765        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
19766        assert_eq!(fallback, c1);
19767
19768        let mut state = review_state(&[
19769            "chore(deps): update a crate",
19770            "fix(daemon): apply a chosen action",
19771        ]);
19772        state.repo = repo.clone();
19773        state.base_branch = "main".to_owned();
19774        state.base_commit = c1;
19775        refresh_reviewed_commits(&mut state, "feat/own").await;
19776        assert_eq!(
19777            state.reviewed_commits,
19778            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
19779        );
19780        assert_eq!(
19781            review_title(&state).as_deref(),
19782            Some("fix(daemon): apply a chosen action")
19783        );
19784        assert_eq!(
19785            leaked_subjects(&state, "feat/own").await,
19786            Some(vec!["chore(deps): update a crate".to_owned()])
19787        );
19788        // A stale tracking ref is still used when the fetch fails, but the
19789        // leak list is withheld.
19790        state.config.merge.remote = "nowhere".to_owned();
19791        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
19792    }
19793
19794    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
19795    /// with `origin/main` one commit behind it, and a summarizing agent that
19796    /// prints whatever `reply` holds.
19797    struct SummaryFixture {
19798        _tmp: tempfile::TempDir,
19799        repo: PathBuf,
19800        reply: PathBuf,
19801        runner: Runner,
19802    }
19803
19804    fn summary_fixture() -> SummaryFixture {
19805        ask_test_home();
19806        let tmp = tempfile::tempdir().unwrap();
19807        let repo = tmp.path().join("repo");
19808        std::fs::create_dir_all(&repo).unwrap();
19809        init_repo(&repo);
19810        let origin = tmp.path().join("origin.git");
19811        let g = |args: &[&str]| {
19812            let out = std::process::Command::new("git")
19813                .args(args)
19814                .current_dir(&repo)
19815                .quiet()
19816                .output()
19817                .expect("spawn git");
19818            assert!(
19819                out.status.success(),
19820                "git {args:?}: {}",
19821                String::from_utf8_lossy(&out.stderr)
19822            );
19823        };
19824        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
19825        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
19826        g(&["push", "-q", "origin", "main"]);
19827        g(&["checkout", "-q", "-b", "feat/own"]);
19828        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19829        g(&["add", "-A"]);
19830        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
19831        let reply = tmp.path().join("reply.txt");
19832        std::fs::write(
19833            &reply,
19834            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
19835        )
19836        .unwrap();
19837        let mut runner = runner_at(RunStatus::Gating);
19838        let mut state = review_state(&["feat(own): add the own file"]);
19839        state.repo = repo.clone();
19840        state.base_branch = "main".to_owned();
19841        state.candidates[0].branch = "feat/own".to_owned();
19842        state.candidates[0].worktree = repo.clone();
19843        let mut author = spec("alpha");
19844        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
19845        author
19846            .env
19847            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
19848        state.config.agents = vec![author];
19849        runner.state = state;
19850        SummaryFixture {
19851            _tmp: tmp,
19852            repo,
19853            reply,
19854            runner,
19855        }
19856    }
19857
19858    async fn posted(f: &mut SummaryFixture) -> PrMessage {
19859        let winner = f.runner.state.candidates[0].clone();
19860        f.runner
19861            .guarded_pr_message(&winner, None, true)
19862            .await
19863            .unwrap()
19864            .expect("an English description never parks")
19865    }
19866
19867    fn clean_round_with_minor_findings() -> ReviewRound {
19868        let mut round = passing_round(2, None);
19869        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
19870        round.reviews = vec![ReviewRecord {
19871            attempts: 0,
19872            reviewer: 1,
19873            agent: "alpha".into(),
19874            summary: "内部レビュー".into(),
19875            findings: vec![
19876                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
19877                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
19878            ],
19879            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
19880            failed: None,
19881            duration_ms: 0,
19882        }];
19883        round
19884    }
19885
19886    #[tokio::test]
19887    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
19888        for review_only in [false, true] {
19889            let mut f = summary_fixture();
19890            if !review_only {
19891                f.runner.state.candidates[0].agent = "alpha".into();
19892            }
19893            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
19894            assert!(f.runner.state.reviews[0].clean);
19895            assert!(f.runner.state.reviews[0].fix.is_none());
19896            assert!(f.runner.state.open_findings().is_empty());
19897            assert_eq!(
19898                crate::pr_outcome::required_finding_ids(&f.runner.state),
19899                ["R2-1-1", "R2-1-2"]
19900            );
19901            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19902            let evidence = crate::pr_outcome::summary_evidence(
19903                &f.runner.state,
19904                crate::pr_outcome::FinalHead::At(&head),
19905            );
19906            assert!(
19907                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
19908            );
19909            // The ordinary reply omits both IDs: no cached description is adopted.
19910            let omitted = posted(&mut f).await;
19911            assert!(omitted.body.contains("No description"));
19912            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
19913            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
19914            assert!(f.runner.state.change_summary.is_none());
19915            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
19916            let described = posted(&mut f).await;
19917            assert!(described.body.contains("improve documentation for callers"));
19918            assert!(described.body.contains("remaining edge case"));
19919            assert!(described.body.contains("## Open review findings"));
19920            assert!(!described.body.contains("未修正"));
19921        }
19922    }
19923
19924    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
19925        let mut f = summary_fixture();
19926        if !review_only {
19927            f.runner.state.candidates[0].agent = "alpha".into();
19928        }
19929        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19930        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
19931        fixed
19932            .fix
19933            .as_mut()
19934            .unwrap()
19935            .rejected
19936            .push(crate::verdict::Rejection {
19937                id: "R1-1-1".into(),
19938                why: "呼び出されないコードのため辞退".into(),
19939            });
19940        let mut clean = clean_round_with_minor_findings();
19941        clean.verified_head = Some(head.clone());
19942        f.runner.state.reviews = vec![fixed, clean];
19943        ok_gate(&mut f.runner.state, Some(&head));
19944        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
19945        f.runner.state.config.agents.push(scripted("beta", reply));
19946        f.runner.state.config.roles.language_judge =
19947            Some(crate::config::AgentChoice::One("beta".into()));
19948        f
19949    }
19950
19951    #[tokio::test]
19952    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
19953        for review_only in [false, true] {
19954            let mut f = rewrite_fixture_with_findings(review_only,
19955                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
19956            let m = posted(&mut f).await;
19957            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19958            let expected =
19959                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
19960            assert!(m.body.contains(&expected));
19961            assert!(m.body.contains("## Review rounds"));
19962            assert!(
19963                m.body
19964                    .contains("Gate (`verify.gate`): passed on the final commit")
19965            );
19966            assert!(
19967                m.body
19968                    .contains("Tests (`verify.e2e`): passed on the final commit")
19969            );
19970            let c = f.runner.state.change_summary.as_ref().unwrap();
19971            assert!(
19972                m.body
19973                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
19974            );
19975            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19976            assert!(crate::pr_outcome::covers_required_findings(
19977                &f.runner.state,
19978                &m.body
19979            ));
19980            assert!(m.body.contains("documentation remains incomplete"));
19981            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
19982            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
19983            assert!(f.runner.state.github_text.is_none());
19984        }
19985    }
19986
19987    #[tokio::test]
19988    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
19989        for review_only in [false, true] {
19990            for reply in [
19991                "TITLE: fix: public description\nA short English description.",
19992                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
19993            ] {
19994                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
19995                let winner = f.runner.state.candidates[0].clone();
19996                let posted = f
19997                    .runner
19998                    .guarded_pr_message(&winner, None, true)
19999                    .await
20000                    .unwrap();
20001                assert!(posted.is_none());
20002                assert!(f.runner.state.parked);
20003                let withheld = f.runner.state.github_text.as_ref().unwrap();
20004                assert!(
20005                    withheld.rewrite.is_none(),
20006                    "an incomplete rewrite must not be selectable"
20007                );
20008                let q = ask::Questions::open()
20009                    .get(withheld.question.as_ref().unwrap())
20010                    .unwrap();
20011                assert!(
20012                    !q.choices
20013                        .contains(&crate::github_text::USE_REWRITE.to_owned())
20014                );
20015                assert!(
20016                    q.detail.contains("R1-1-1")
20017                        && q.detail.contains("R2-1-1")
20018                        && q.detail.contains("R2-1-2")
20019                );
20020            }
20021        }
20022    }
20023
20024    #[tokio::test]
20025    async fn japanese_reports_stay_local_in_both_public_run_modes() {
20026        for review_only in [false, true] {
20027            let mut f = summary_fixture();
20028            if !review_only {
20029                f.runner.state.candidates[0].agent = "alpha".into();
20030            }
20031            f.runner.state.instruction = "内部タスクの説明".into();
20032            f.runner.state.candidates[0].summary = "内部の候補報告".into();
20033            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20034            f.runner.state.reviews = vec![
20035                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
20036                round_with_notes(2, Some("二回目の内部修正報告です。")),
20037                passing_round(3, Some(&head)),
20038            ];
20039            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
20040            ok_gate(&mut f.runner.state, Some(&head));
20041            let captured = f.repo.join("public-prompt.txt");
20042            let author = &mut f.runner.state.config.agents[0];
20043            author.command = vec![
20044                "sh".into(),
20045                "-c".into(),
20046                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
20047            ];
20048            author.env.insert(
20049                "PROMPT_FILE".into(),
20050                captured.to_string_lossy().into_owned(),
20051            );
20052            let m = posted(&mut f).await;
20053            assert!(m.body.contains("Adds `own.txt`"));
20054            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
20055            assert!(m.body.contains("R1-1-1"));
20056            assert!(
20057                m.body
20058                    .contains("Gate (`verify.gate`): passed on the final commit")
20059            );
20060            assert!(
20061                m.body
20062                    .contains("Tests (`verify.e2e`): passed on the final commit")
20063            );
20064            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
20065            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
20066            let prompt = std::fs::read_to_string(captured).unwrap();
20067            assert!(prompt.contains(&format!(
20068                "git diff {} {head}",
20069                f.runner.state.change_summary.as_ref().unwrap().base
20070            )));
20071            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
20072            assert_eq!(
20073                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
20074                "二回目の内部修正報告です。"
20075            );
20076        }
20077    }
20078
20079    #[tokio::test]
20080    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
20081        for review_only in [false, true] {
20082            let mut f = summary_fixture();
20083            if !review_only {
20084                f.runner.state.candidates[0].agent = "alpha".into();
20085            }
20086            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
20087            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
20088            std::fs::write(&f.reply, "no usable description").unwrap();
20089            let failed = posted(&mut f).await;
20090            assert!(
20091                failed
20092                    .body
20093                    .contains("No description of what this change does could be generated")
20094            );
20095            assert!(failed.body.is_ascii());
20096            assert!(f.runner.state.github_text.is_none());
20097            assert!(f.runner.state.change_summary.is_none());
20098            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20099            let evidence = crate::pr_outcome::summary_evidence(
20100                &f.runner.state,
20101                crate::pr_outcome::FinalHead::At(&head),
20102            );
20103            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
20104            assert!(
20105                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
20106                    .exists()
20107            );
20108            f.runner.state =
20109                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
20110            std::fs::write(
20111                &f.reply,
20112                "TITLE: feat(own): add the own file\nAdds the committed own file.",
20113            )
20114            .unwrap();
20115            let retried = posted(&mut f).await;
20116            assert!(retried.body.contains("Adds the committed own file."));
20117            assert_eq!(
20118                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
20119                "日本語の修正報告"
20120            );
20121        }
20122    }
20123
20124    #[tokio::test]
20125    async fn same_head_verification_and_findings_invalidate_public_prose() {
20126        let mut f = summary_fixture();
20127        posted(&mut f).await;
20128        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20129        let original = f
20130            .runner
20131            .state
20132            .change_summary
20133            .as_ref()
20134            .unwrap()
20135            .evidence
20136            .clone();
20137        ok_gate(&mut f.runner.state, Some(&head));
20138        std::fs::write(
20139            &f.reply,
20140            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
20141        )
20142        .unwrap();
20143        let verified = posted(&mut f).await;
20144        assert!(verified.body.contains("now verified"));
20145        assert_ne!(
20146            f.runner.state.change_summary.as_ref().unwrap().evidence,
20147            original
20148        );
20149        let mut round = round_with_notes(1, Some("内部報告"));
20150        round.reviews.push(ReviewRecord {
20151            attempts: 0,
20152            reviewer: 1,
20153            agent: "alpha".into(),
20154            summary: "内部レビュー".into(),
20155            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
20156            vote: None,
20157            failed: None,
20158            duration_ms: 0,
20159        });
20160        round
20161            .fix
20162            .as_mut()
20163            .unwrap()
20164            .rejected
20165            .push(crate::verdict::Rejection {
20166                id: "R1-1-1".into(),
20167                why: "呼び出し元から到達できない".into(),
20168            });
20169        f.runner.state.reviews = vec![round];
20170        // A description omitting the required ID is unusable, even with a title.
20171        let omitted = posted(&mut f).await;
20172        assert!(omitted.body.contains("No description"));
20173        assert!(omitted.body.contains("R1-1-1"));
20174        assert!(!omitted.body.contains("到達"));
20175        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
20176        let described = posted(&mut f).await;
20177        assert!(described.body.contains("unreachable from callers"));
20178        assert!(described.body.contains("R1-1-1"));
20179        assert!(!described.body.contains("到達"));
20180        let evidence = f
20181            .runner
20182            .state
20183            .change_summary
20184            .as_ref()
20185            .unwrap()
20186            .evidence
20187            .clone();
20188        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
20189        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
20190        assert!(posted(&mut f).await.body.contains("rationale updated"));
20191        assert_ne!(
20192            f.runner.state.change_summary.as_ref().unwrap().evidence,
20193            evidence
20194        );
20195        // Old serialized summaries have no evidence fingerprint and must regenerate.
20196        f.runner
20197            .state
20198            .change_summary
20199            .as_mut()
20200            .unwrap()
20201            .evidence
20202            .clear();
20203        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
20204        assert!(posted(&mut f).await.body.contains("current specification"));
20205    }
20206
20207    #[tokio::test]
20208    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
20209        let mut f = summary_fixture();
20210        let m = posted(&mut f).await;
20211        assert_eq!(m.title, "feat(own): add the own file");
20212        assert!(
20213            m.body
20214                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
20215            "{}",
20216            m.body
20217        );
20218        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
20219        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
20220        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
20221        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20222        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
20223        let kept = f.runner.state.change_summary.as_ref().unwrap();
20224        assert_eq!(
20225            (kept.head.as_str(), kept.base.as_str()),
20226            (head.as_str(), base.as_str())
20227        );
20228        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
20229    }
20230
20231    #[tokio::test]
20232    async fn review_only_summary_receives_the_configured_github_language() {
20233        let mut f = summary_fixture();
20234        f.runner.state.config.graph.github_language = "ja".into();
20235        let captured = f.repo.join("summary-prompt.txt");
20236        let author = &mut f.runner.state.config.agents[0];
20237        author.command = vec![
20238            "sh".into(),
20239            "-c".into(),
20240            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
20241        ];
20242        author.env.insert(
20243            "PROMPT_FILE".into(),
20244            captured.to_string_lossy().into_owned(),
20245        );
20246        std::fs::write(
20247            &f.reply,
20248            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
20249        )
20250        .unwrap();
20251        let m = posted(&mut f).await;
20252        let prompt = std::fs::read_to_string(captured).unwrap();
20253        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
20254        assert!(!prompt.contains("in English"));
20255        assert_eq!(m.title, "feat(own): ファイルを追加");
20256        assert!(m.body.contains("own.txt を追加します。"));
20257    }
20258
20259    #[tokio::test]
20260    async fn review_fixes_come_after_the_whole_change_summary() {
20261        let mut f = summary_fixture();
20262        f.runner.state.reviews = vec![
20263            round_with_notes(1, Some("handled the PATH case")),
20264            round_with_notes(2, None),
20265        ];
20266        let m = posted(&mut f).await;
20267        let summary = m.body.find("## Summary").expect("summary");
20268        let fixes = m.body.find("## Review rounds").expect("review rounds");
20269        assert!(summary < fixes, "{}", m.body);
20270        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
20271        assert!(m.body.contains("## Final verification"), "{}", m.body);
20272    }
20273
20274    #[tokio::test]
20275    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
20276        let mut f = summary_fixture();
20277        let first = posted(&mut f).await;
20278        let first_head = f.runner.state.change_summary.clone().unwrap().head;
20279        // The same head reuses the record: a different reply is not read.
20280        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
20281        let again = posted(&mut f).await;
20282        assert_eq!((again.title, again.body), (first.title, first.body));
20283
20284        let g = |args: &[&str]| {
20285            let out = std::process::Command::new("git")
20286                .args(args)
20287                .current_dir(&f.repo)
20288                .quiet()
20289                .output()
20290                .expect("spawn git");
20291            assert!(out.status.success(), "git {args:?}");
20292        };
20293        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
20294        g(&["add", "-A"]);
20295        g(&["commit", "-q", "-m", "fix(own): handle more"]);
20296        let second = posted(&mut f).await;
20297        let kept = f.runner.state.change_summary.clone().unwrap();
20298        assert_ne!(kept.head, first_head);
20299        assert!(second.body.contains("Not read yet."), "{}", second.body);
20300        assert!(!second.body.contains("so the first summary is read"));
20301        assert_eq!(second.title, "feat(own): other");
20302    }
20303
20304    #[tokio::test]
20305    async fn an_unavailable_summary_is_stated_and_never_invented() {
20306        let mut f = summary_fixture();
20307        std::fs::write(&f.reply, "no title line here").unwrap();
20308        let m = posted(&mut f).await;
20309        assert!(
20310            m.body
20311                .contains("No description of what this change does could be generated"),
20312            "{}",
20313            m.body
20314        );
20315        assert!(m.body.contains("internal reports remain in local records"));
20316        assert!(f.runner.state.change_summary.is_none());
20317        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
20318        // A preview asks nobody and claims nothing.
20319        let winner = f.runner.state.candidates[0].clone();
20320        let preview = f
20321            .runner
20322            .guarded_pr_message(&winner, None, false)
20323            .await
20324            .unwrap()
20325            .unwrap();
20326        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
20327    }
20328
20329    #[test]
20330    fn pr_message_review_single_commit_uses_its_subject() {
20331        let state = review_state(&["feat(nats): per-role user"]);
20332        let m = pr_message(&state, 'A');
20333        assert_eq!(m.title, "feat(nats): per-role user");
20334        assert!(!m.body.contains("Review the work already"), "{}", m.body);
20335        assert!(m.body.contains("## Commits under review"), "{}", m.body);
20336    }
20337
20338    #[test]
20339    fn pr_message_review_multi_commit_takes_the_oldest() {
20340        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
20341        let m = pr_message(&state, 'A');
20342        assert_eq!(m.title, "feat: the change");
20343        for s in ["feat: the change", "fix: typo", "fix: again"] {
20344            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
20345        }
20346    }
20347
20348    #[test]
20349    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
20350        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
20351            let mut state = review_state(&[first, "fix: later fixup"]);
20352            state.candidates[0].branch = "機能/ログイン".to_owned();
20353            let m = pr_message(&state, 'A');
20354            assert!(
20355                m.title.starts_with("chore: land candidate A of run"),
20356                "{}",
20357                m.title
20358            );
20359        }
20360    }
20361
20362    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
20363        BranchFacts {
20364            commits: commits
20365                .iter()
20366                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
20367                .collect(),
20368            stat: stat.to_owned(),
20369        }
20370    }
20371
20372    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
20373        let mut r = review_round(true, 0, 1, 1, true, true);
20374        r.round = round;
20375        r.fix = notes.map(|n| FixRecord {
20376            agent: "fixer".to_owned(),
20377            addressed: Vec::new(),
20378            rejected: Vec::new(),
20379            notes: n.to_owned(),
20380            committed: true,
20381            failed: None,
20382            duration_ms: 0,
20383            continuation: None,
20384        });
20385        r
20386    }
20387
20388    #[test]
20389    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
20390        let state = review_state(&["ignored"]);
20391        let f = facts(
20392            &[
20393                (
20394                    "fix(login): resolve PATH on macOS",
20395                    "Login shells skip rc files.",
20396                ),
20397                ("fix: address review", ""),
20398            ],
20399            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
20400        );
20401        let m = pr_message_with(&state, 'A', Some(&f));
20402        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
20403        assert!(
20404            m.body
20405                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
20406            "{}",
20407            m.body
20408        );
20409        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
20410        assert!(m.body.contains("## Diff stat"), "{}", m.body);
20411        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
20412        for banned in [
20413            "Review the work already",
20414            "no task statement",
20415            "Original task",
20416        ] {
20417            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
20418        }
20419        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
20420    }
20421
20422    #[test]
20423    fn pr_message_review_truncates_a_huge_first_commit_body() {
20424        let state = review_state(&["ignored"]);
20425        let f = facts(
20426            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
20427            "s",
20428        );
20429        let m = pr_message_with(&state, 'A', Some(&f));
20430        assert!(m.body.len() < 30_000, "{}", m.body.len());
20431        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
20432        assert!(
20433            m.body.contains("- ... 1 more commit(s)"),
20434            "{}",
20435            m.body.len()
20436        );
20437        assert!(m.body.ends_with("magi:candidate-a\n"));
20438    }
20439
20440    #[test]
20441    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
20442        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
20443        assert_eq!(m.title, "feat: x");
20444        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
20445        assert!(!m.body.contains("Diff stat"), "{}", m.body);
20446        assert!(!m.body.contains("no task statement"), "{}", m.body);
20447    }
20448
20449    #[test]
20450    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
20451        let mut state = review_state(&["日本語の件名"]);
20452        state.candidates[0].branch = "fix/macos-login-path".to_owned();
20453        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
20454        state.candidates[0].branch = "機能/ログイン".to_owned();
20455        assert!(
20456            pr_message(&state, 'A')
20457                .title
20458                .starts_with("chore: land candidate A")
20459        );
20460    }
20461
20462    #[test]
20463    fn pr_message_review_fixes_survive_a_clean_final_round() {
20464        let mut state = review_state(&["feat: x"]);
20465        state.reviews = vec![
20466            round_with_notes(1, Some("handled the PATH case")),
20467            round_with_notes(2, None),
20468        ];
20469        let body = pr_message(&state, 'A').body;
20470        assert!(
20471            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
20472            "{body}"
20473        );
20474        assert!(body.contains("### Round 1"), "{body}");
20475
20476        state.reviews = vec![
20477            round_with_notes(1, Some("first fix")),
20478            round_with_notes(2, Some("")),
20479            round_with_notes(3, Some("second fix")),
20480            round_with_notes(4, None),
20481        ];
20482        let body = pr_message(&state, 'A').body;
20483        assert!(
20484            body.contains("### Round 1") && !body.contains("first fix"),
20485            "{body}"
20486        );
20487        assert!(
20488            body.contains("### Round 3") && !body.contains("second fix"),
20489            "{body}"
20490        );
20491        assert!(body.contains("### Round 2"), "{body}");
20492    }
20493
20494    const FINAL: &str = "abcdef1234567890";
20495
20496    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
20497        let mut r = round_with_notes(round, None);
20498        r.verified_head = verified.map(str::to_owned);
20499        r
20500    }
20501
20502    fn ok_gate(state: &mut RunState, head: Option<&str>) {
20503        state.gate = vec![CommandOutcome {
20504            command: "gate".to_owned(),
20505            code: Some(0),
20506            output_tail: String::new(),
20507            duration_ms: 0,
20508            resource_blocked: false,
20509        }];
20510        state.gate_ran = true;
20511        state.gate_head = head.map(str::to_owned);
20512    }
20513
20514    fn body_at(state: &RunState) -> String {
20515        pr_message_raw(
20516            state,
20517            'A',
20518            None,
20519            crate::pr_outcome::FinalHead::At(FINAL),
20520            ChangeNote::NotAsked,
20521        )
20522        .body
20523    }
20524
20525    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
20526
20527    #[test]
20528    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
20529        let mut state = review_state(&["feat: x"]);
20530        state.reviews = vec![
20531            round_with_notes(1, Some(SANDBOXED)),
20532            passing_round(2, Some(FINAL)),
20533        ];
20534        ok_gate(&mut state, Some(FINAL));
20535        let body = body_at(&state);
20536        assert!(!body.contains("Handled the PATH case."), "{body}");
20537        assert!(!body.contains("could not commit"), "{body}");
20538        assert!(!body.contains("could not run"), "{body}");
20539        assert!(
20540            body.contains("The fix is committed on the branch"),
20541            "{body}"
20542        );
20543        assert!(!body.contains("sandbox"), "{body}");
20544        assert!(
20545            body.contains("Tests (`verify.e2e`): passed on the final commit"),
20546            "{body}"
20547        );
20548        assert!(
20549            body.contains("Gate (`verify.gate`): passed on the final commit"),
20550            "{body}"
20551        );
20552        // The raw report is untouched in the record.
20553        assert!(
20554            state.reviews[0]
20555                .fix
20556                .as_ref()
20557                .unwrap()
20558                .notes
20559                .contains("could not commit")
20560        );
20561    }
20562
20563    #[test]
20564    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
20565        let mut state = review_state(&["feat: x"]);
20566        let mut r1 = round_with_notes(1, Some(SANDBOXED));
20567        r1.fix.as_mut().unwrap().committed = false;
20568        let mut r2 = passing_round(2, Some(FINAL));
20569        r2.e2e[0].code = Some(1);
20570        state.reviews = vec![r1, r2];
20571        let body = body_at(&state);
20572        assert!(!body.contains("could not commit"), "{body}");
20573        assert!(!body.contains("could not run"), "{body}");
20574        assert!(body.contains("This round produced no commit"), "{body}");
20575        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
20576        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
20577        assert!(!body.contains("passed"), "{body}");
20578    }
20579
20580    #[test]
20581    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
20582        let mut state = review_state(&["feat: x"]);
20583        state.reviews = vec![
20584            round_with_notes(1, Some(SANDBOXED)),
20585            passing_round(2, Some("0123456789abcdef")),
20586        ];
20587        ok_gate(&mut state, Some("0123456789abcdef"));
20588        let body = body_at(&state);
20589        assert!(!body.contains("HTTP tests could not run"), "{body}");
20590        // The commit claim is still settled by the structured record.
20591        assert!(!body.contains("could not commit"), "{body}");
20592        assert!(
20593            body.contains("passed on an earlier commit (`0123456`)"),
20594            "{body}"
20595        );
20596        assert!(!body.contains("passed on the final commit"), "{body}");
20597
20598        // A gate recorded before `gate_head` existed is "not recorded".
20599        ok_gate(&mut state, None);
20600        let body = body_at(&state);
20601        assert!(
20602            body.contains("passed on a commit that was not recorded"),
20603            "{body}"
20604        );
20605    }
20606
20607    #[test]
20608    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
20609        let state = state_with_summary(
20610            "add retries to the client",
20611            "TITLE: feat: retries\n\nDid it.",
20612        );
20613        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
20614        assert_eq!(m.title, "feat: retries");
20615        assert!(
20616            m.body.contains("<summary>Original task</summary>"),
20617            "{}",
20618            m.body
20619        );
20620        assert!(!m.body.contains("Commits under review"), "{}", m.body);
20621        assert!(
20622            m.body
20623                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
20624            "{}",
20625            m.body
20626        );
20627    }
20628
20629    #[test]
20630    fn pr_message_review_bounds_a_long_english_subject() {
20631        let long = format!("feat: {}", "word ".repeat(100));
20632        let m = pr_message(&review_state(&[&long]), 'A');
20633        assert!(m.title.starts_with("feat: word"), "{}", m.title);
20634        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20635    }
20636
20637    #[test]
20638    fn pr_message_implementation_run_is_unchanged_by_review_support() {
20639        let state = state_with_summary("add retries\n\ndetails", "- did some things");
20640        let m = pr_message(&state, 'A');
20641        assert_eq!(m.title, "add retries");
20642        assert!(m.body.contains("<summary>Original task</summary>"));
20643        assert!(!m.body.contains("Commits under review"));
20644        assert_eq!(landing_subject_source(&state), state.instruction);
20645    }
20646
20647    #[test]
20648    fn review_run_squash_subject_is_the_change_not_the_prompt() {
20649        let state = review_state(&["feat: the change", "fix: typo"]);
20650        let source = landing_subject_source(&state);
20651        assert_eq!(land::merge_subject("", &source), "feat: the change");
20652        assert_eq!(
20653            land::merge_subject("magi: candidate A (uncommitted work)", &source),
20654            "feat: the change"
20655        );
20656        // An operator's rename still wins.
20657        assert_eq!(
20658            land::merge_subject("feat: renamed by hand", &source),
20659            "feat: renamed by hand"
20660        );
20661        let blank = review_state(&["日本語"]);
20662        assert!(
20663            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
20664        );
20665    }
20666
20667    #[test]
20668    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
20669        let state = review_state(&["feat: the change"]);
20670        let source = landing_subject_source(&state);
20671        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
20672        assert_eq!(
20673            land::merge_subject(landing_title(&state, old), &source),
20674            "feat: the change"
20675        );
20676        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
20677        let task = state_with_summary("add retries", "");
20678        assert_eq!(landing_title(&task, old), old);
20679    }
20680
20681    #[test]
20682    fn pr_message_describes_the_change_not_the_task() {
20683        let state = state_with_summary(
20684            "今回やってほしいこと: results projector を直す",
20685            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
20686        );
20687        let m = pr_message(&state, 'A');
20688        assert_eq!(m.title, "fix(web): batch the runs list reads");
20689        assert!(
20690            m.body.starts_with("## Summary\n\n- reads run.json once"),
20691            "{}",
20692            m.body
20693        );
20694        assert!(!m.body.contains("TITLE:"), "{}", m.body);
20695        let task_at = m.body.find("今回やってほしいこと").unwrap();
20696        let details_at = m.body.find("<details>").unwrap();
20697        assert!(
20698            details_at < task_at,
20699            "the task lives inside <details>: {}",
20700            m.body
20701        );
20702        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
20703        assert!(m.body.contains("magi:candidate-a"));
20704    }
20705
20706    #[test]
20707    fn pr_message_falls_back_to_the_task_without_a_title_line() {
20708        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
20709        let m = pr_message(&state, 'A');
20710        assert_eq!(m.title, "add retries");
20711        assert!(
20712            m.body.contains("## Summary\n\n- did some things"),
20713            "{}",
20714            m.body
20715        );
20716
20717        let none = RunState::new(
20718            PathBuf::from("/repo"),
20719            "main".to_owned(),
20720            "abc1234".to_owned(),
20721            "add retries".to_owned(),
20722            Config::default(),
20723        );
20724        let m = pr_message(&none, 'A');
20725        assert_eq!(m.title, "add retries");
20726        assert!(!m.body.contains("## Summary"), "{}", m.body);
20727    }
20728
20729    #[test]
20730    fn pr_message_refuses_the_candidate_commit_subject() {
20731        for bad in [
20732            "TITLE: magi: candidate A (uncommitted work)",
20733            "TITLE: chore: stuff (uncommitted work)",
20734            "TITLE:   ",
20735        ] {
20736            let state = state_with_summary("add retries", bad);
20737            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
20738        }
20739    }
20740
20741    #[test]
20742    fn pr_message_bounds_a_very_long_task_and_title() {
20743        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
20744        let state = state_with_summary(&long, "- nothing");
20745        let m = pr_message(&state, 'A');
20746        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20747        assert!(!m.title.contains('\n'));
20748
20749        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
20750        let m = pr_message(&state, 'A');
20751        assert!(m.title.starts_with("feat: "));
20752        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20753        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
20754    }
20755
20756    fn long_title_of(instruction: &str) -> String {
20757        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
20758    }
20759
20760    #[test]
20761    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
20762        let first = "Make the landing path keep a readable title for long tasks";
20763        let line = format!(
20764            "{first}. {}",
20765            "And then keep going with more words ".repeat(20)
20766        );
20767        let t = long_title_of(&line);
20768        assert_eq!(t, first);
20769        assert!(!t.starts_with("chore: land"));
20770    }
20771
20772    #[test]
20773    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
20774        let line = "word ".repeat(200);
20775        let t = long_title_of(&line);
20776        assert!(t.ends_with("word..."), "{t}");
20777        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
20778    }
20779
20780    #[test]
20781    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
20782        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
20783            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
20784        }
20785    }
20786
20787    #[test]
20788    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
20789        let mut state = state_with_summary("再試行を追加する", "- nothing");
20790        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
20791        state.config.graph.github_language = "ja".into();
20792        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
20793    }
20794
20795    #[test]
20796    fn pr_message_title_limit_is_exact() {
20797        let at = "a".repeat(PR_TITLE_MAX);
20798        assert_eq!(long_title_of(&at), at);
20799        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
20800        assert!(over.ends_with("..."), "{over}");
20801        assert_eq!(over.chars().count(), PR_TITLE_MAX);
20802    }
20803
20804    #[test]
20805    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
20806        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
20807        let t = long_title_of(&line);
20808        assert!(t.ends_with("..."), "{t}");
20809        assert!(t.is_ascii(), "{t}");
20810    }
20811
20812    #[test]
20813    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
20814        let line = format!(
20815            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
20816            "plus more filler words ".repeat(20)
20817        );
20818        let t = long_title_of(&line);
20819        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
20820    }
20821
20822    #[test]
20823    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
20824        let line = format!("\n\n# {}", "title words ".repeat(40));
20825        let state = state_with_summary(&line, "- nothing");
20826        let m = pr_message(&state, 'A');
20827        assert!(m.title.starts_with("title words"), "{}", m.title);
20828        assert_eq!(
20829            land::merge_subject(&m.title, &landing_subject_source(&state)),
20830            m.title
20831        );
20832        // An operator's rename wins untouched.
20833        assert_eq!(
20834            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
20835            "feat: renamed by hand"
20836        );
20837    }
20838
20839    #[tokio::test]
20840    async fn github_text_rewrite_is_bounded_and_falls_back() {
20841        crate::run::pin_test_home();
20842        for (reply, accepted) in [
20843            (
20844                "TITLE: fix: retries\nAdd retries for failed requests.",
20845                true,
20846            ),
20847            (
20848                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
20849                false,
20850            ),
20851            (
20852                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
20853                true,
20854            ),
20855            (
20856                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
20857                true,
20858            ),
20859            (
20860                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
20861                true,
20862            ),
20863            (
20864                "TITLE: fix: retries\n日本語の説明です。 token=secret",
20865                false,
20866            ),
20867            ("TITLE: fix: retries\n", false),
20868        ] {
20869            let dir = tempfile::tempdir().unwrap();
20870            let mut runner = runner_at(RunStatus::Gating);
20871            runner.state = state_with_summary(
20872                "add retries",
20873                "TITLE: fix: retries\n日本語の説明を書きます。",
20874            );
20875            runner.state.repo = dir.path().to_owned();
20876            runner.state.candidates[0].worktree = dir.path().to_owned();
20877            let mut author = spec("alpha");
20878            author.command = vec![
20879                "sh".into(),
20880                "-c".into(),
20881                "printf '%s' \"$REWRITE_REPLY\"".into(),
20882            ];
20883            author.env.insert("REWRITE_REPLY".into(), reply.into());
20884            runner.state.config.agents = vec![author];
20885            seed_public_summary(&mut runner);
20886            let winner = runner.state.candidates[0].clone();
20887            let first = runner
20888                .guarded_pr_message(&winner, None, true)
20889                .await
20890                .unwrap();
20891            let message = if accepted {
20892                first.expect("a passing rewrite never asks")
20893            } else {
20894                // Withheld: the run parks on a question and posts nothing yet.
20895                assert!(first.is_none());
20896                assert!(runner.state.parked);
20897                let store = ask::Questions::open();
20898                let asked: Vec<_> = store
20899                    .list()
20900                    .into_iter()
20901                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20902                    .collect();
20903                assert_eq!(asked.len(), 1);
20904                assert!(!asked[0].detail.contains("secret"));
20905                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
20906                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
20907                assert_eq!(
20908                    asked[0]
20909                        .choices
20910                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
20911                    rewrite.is_some()
20912                );
20913                // Silence: the question lapses and the neutral text is used,
20914                // without a second rewrite call or a second question.
20915                store
20916                    .update(&asked[0].id, |q| {
20917                        q.abandon("test");
20918                        Ok(())
20919                    })
20920                    .unwrap();
20921                let again = runner
20922                    .guarded_pr_message(&winner, None, true)
20923                    .await
20924                    .unwrap()
20925                    .expect("an abandoned question falls back");
20926                let n = store
20927                    .list()
20928                    .into_iter()
20929                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20930                    .count();
20931                assert_eq!(n, 1, "asked once per rejected text");
20932                again
20933            };
20934            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
20935            assert!(!message.body.contains("secret"));
20936            assert!(!message.body.contains("prefix"));
20937            assert!(!message.title.contains("secret"));
20938            assert_eq!(
20939                message.body.contains("Add retries for failed requests."),
20940                accepted
20941            );
20942            assert_eq!(
20943                runner.state.seats["github-text-rewrite"].turns, 1,
20944                "only one rewrite invocation"
20945            );
20946            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
20947            if accepted {
20948                assert!(runner.state.github_text.is_none());
20949                assert!(!runner.state.parked);
20950                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
20951                assert!(message.body.contains("## Final verification"));
20952                assert!(message.body.contains("Evidence: final diff"));
20953                assert!(
20954                    message
20955                        .body
20956                        .contains(&format!("magi:run/{}", runner.state.id))
20957                );
20958            }
20959            if !accepted {
20960                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20961                assert!(
20962                    message
20963                        .body
20964                        .contains(&format!("magi:run/{}", runner.state.id))
20965                );
20966            }
20967        }
20968    }
20969
20970    fn scripted(id: &str, reply: &str) -> AgentSpec {
20971        let mut a = spec(id);
20972        a.command = vec![
20973            "sh".into(),
20974            "-c".into(),
20975            "printf '%s' \"$REWRITE_REPLY\"".into(),
20976        ];
20977        a.env.insert("REWRITE_REPLY".into(), reply.into());
20978        a
20979    }
20980
20981    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
20982    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
20983
20984    /// Gate tests inject generated public output, not internal reports.
20985    fn seed_public_summary(runner: &mut Runner) {
20986        let repo = &runner.state.repo;
20987        let g = |args: &[&str]| {
20988            let out = std::process::Command::new("git")
20989                .args(args)
20990                .current_dir(repo)
20991                .quiet()
20992                .output()
20993                .unwrap();
20994            assert!(out.status.success(), "git {args:?}");
20995            String::from_utf8_lossy(&out.stdout).trim().to_owned()
20996        };
20997        g(&["init", "-q", "-b", "main"]);
20998        g(&[
20999            "-c",
21000            "user.name=Test",
21001            "-c",
21002            "user.email=test@example.invalid",
21003            "commit",
21004            "-q",
21005            "--allow-empty",
21006            "-m",
21007            "feat: fixture",
21008        ]);
21009        let head = g(&["rev-parse", "HEAD"]);
21010        g(&["update-ref", "refs/remotes/origin/main", &head]);
21011        let evidence = crate::pr_outcome::summary_evidence(
21012            &runner.state,
21013            crate::pr_outcome::FinalHead::At(&head),
21014        );
21015        let generated = &runner.state.candidates[0].summary;
21016        runner.state.change_summary = Some(crate::run::ChangeSummary {
21017            base: head.clone(),
21018            head,
21019            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
21020            title: summary_title(generated).unwrap(),
21021            body: summary_without_title(generated),
21022        });
21023    }
21024
21025    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
21026        let mut runner = runner_at(RunStatus::Gating);
21027        runner.state = state_with_summary("add retries", JA_SUMMARY);
21028        if review_only {
21029            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21030        }
21031        runner.state.repo = dir.to_owned();
21032        runner.state.candidates[0].worktree = dir.to_owned();
21033        runner.state.config.agents = agents;
21034        seed_public_summary(&mut runner);
21035        runner
21036    }
21037
21038    fn gate_questions(runner: &Runner) -> usize {
21039        ask::Questions::open()
21040            .list()
21041            .into_iter()
21042            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
21043            .count()
21044    }
21045
21046    #[tokio::test]
21047    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
21048        crate::run::pin_test_home();
21049        let dir = tempfile::tempdir().unwrap();
21050        let mut runner = rewrite_runner(
21051            dir.path(),
21052            true,
21053            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
21054        );
21055        runner.state.config.roles.language_judge =
21056            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
21057        let winner = runner.state.candidates[0].clone();
21058        let msg = runner
21059            .guarded_pr_message(&winner, None, true)
21060            .await
21061            .unwrap()
21062            .expect("an English rewrite posts without asking");
21063        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
21064        assert!(msg.body.contains("<details>kept</details>"));
21065        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
21066        assert_eq!(gate_questions(&runner), 0);
21067        assert!(runner.state.github_text.is_none());
21068        assert!(!runner.state.seats.contains_key("impl-A"));
21069        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21070    }
21071
21072    #[tokio::test]
21073    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
21074        crate::run::pin_test_home();
21075        let dir = tempfile::tempdir().unwrap();
21076        let mut runner = rewrite_runner(
21077            dir.path(),
21078            false,
21079            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
21080        );
21081        runner.state.config.roles.language_judge =
21082            Some(crate::config::AgentChoice::One("beta".into()));
21083        runner.state.seats.insert(
21084            "impl-A".into(),
21085            SeatState::new("impl-A", "alpha", runner.state.seed),
21086        );
21087        let winner = runner.state.candidates[0].clone();
21088        let msg = runner
21089            .guarded_pr_message(&winner, None, true)
21090            .await
21091            .unwrap()
21092            .expect("rewritten");
21093        assert!(msg.body.contains("<details>kept</details>"));
21094        assert_eq!(runner.state.seats["impl-A"].turns, 0);
21095        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21096        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
21097    }
21098
21099    #[tokio::test]
21100    async fn rewrite_reuses_a_resumable_implementer_session() {
21101        crate::run::pin_test_home();
21102        let dir = tempfile::tempdir().unwrap();
21103        let mut runner = rewrite_runner(
21104            dir.path(),
21105            false,
21106            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
21107        );
21108        runner.state.config.roles.language_judge =
21109            Some(crate::config::AgentChoice::One("beta".into()));
21110        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
21111        seat.turns = 1;
21112        runner.state.seats.insert("impl-A".into(), seat);
21113        let winner = runner.state.candidates[0].clone();
21114        runner
21115            .guarded_pr_message(&winner, None, true)
21116            .await
21117            .unwrap()
21118            .expect("rewritten");
21119        assert_eq!(runner.state.seats["impl-A"].turns, 2);
21120        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
21121    }
21122
21123    #[tokio::test]
21124    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
21125        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
21126        // session, was out of quota, and nothing else was asked.
21127        crate::run::pin_test_home();
21128        let dir = tempfile::tempdir().unwrap();
21129        let mut exhausted = spec("alpha");
21130        exhausted.command = vec!["false".into()];
21131        let mut runner = rewrite_runner(
21132            dir.path(),
21133            false,
21134            vec![exhausted, scripted("beta", EN_REWRITE)],
21135        );
21136        // The implementer's own agent is in the role too: it is not asked twice.
21137        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
21138            "alpha".into(),
21139            "beta".into(),
21140        ]));
21141        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
21142        seat.turns = 1;
21143        runner.state.seats.insert("impl-A".into(), seat);
21144        let winner = runner.state.candidates[0].clone();
21145        let msg = runner
21146            .guarded_pr_message(&winner, None, true)
21147            .await
21148            .unwrap()
21149            .expect("the role rewrites after the implementer fails");
21150        assert!(msg.body.contains("`src/net.rs`"));
21151        assert_eq!(
21152            runner.state.seats["impl-A"].turns, 2,
21153            "the implementer was tried once"
21154        );
21155        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
21156        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21157    }
21158
21159    #[tokio::test]
21160    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
21161        crate::run::pin_test_home();
21162        let dir = tempfile::tempdir().unwrap();
21163        let mut broken = spec("bad");
21164        broken.command = vec!["false".into()];
21165        let mut runner = rewrite_runner(
21166            dir.path(),
21167            true,
21168            vec![
21169                broken,
21170                scripted("noformat", "Add retries."),
21171                scripted("good", EN_REWRITE),
21172            ],
21173        );
21174        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
21175            "bad".into(),
21176            "noformat".into(),
21177            "good".into(),
21178        ]));
21179        let winner = runner.state.candidates[0].clone();
21180        let msg = runner
21181            .guarded_pr_message(&winner, None, true)
21182            .await
21183            .unwrap()
21184            .expect("the third agent rewrites");
21185        assert!(msg.body.contains("`src/net.rs`"));
21186        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
21187        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
21188    }
21189
21190    #[tokio::test]
21191    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
21192        crate::run::pin_test_home();
21193        // (agents, role): still Japanese; no TITLE; nothing resolvable.
21194        for (agents, role, saved) in [
21195            (
21196                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
21197                Some("beta"),
21198                true,
21199            ),
21200            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
21201            (Vec::new(), None, false),
21202        ] {
21203            let dir = tempfile::tempdir().unwrap();
21204            let mut runner = rewrite_runner(dir.path(), true, agents);
21205            runner.state.config.roles.language_judge =
21206                role.map(|r| crate::config::AgentChoice::One(r.into()));
21207            let winner = runner.state.candidates[0].clone();
21208            let first = runner
21209                .guarded_pr_message(&winner, None, true)
21210                .await
21211                .unwrap();
21212            assert!(first.is_none(), "withheld text parks the run");
21213            assert!(runner.state.parked);
21214            assert_eq!(gate_questions(&runner), 1);
21215            let g = runner.state.github_text.as_ref().unwrap();
21216            assert_eq!(g.rewrite.is_some(), saved);
21217            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
21218        }
21219    }
21220
21221    #[tokio::test]
21222    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
21223        crate::run::pin_test_home();
21224        let dir = tempfile::tempdir().unwrap();
21225        let mut runner = runner_at(RunStatus::Gating);
21226        runner.state =
21227            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
21228        runner.state.repo = dir.path().to_owned();
21229        runner.state.candidates[0].worktree = dir.path().to_owned();
21230        let mut author = spec("alpha");
21231        author.command = vec![
21232            "sh".into(),
21233            "-c".into(),
21234            "printf '%s' \"$REWRITE_REPLY\"".into(),
21235        ];
21236        author.env.insert(
21237            "REWRITE_REPLY".into(),
21238            format!(
21239                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
21240                "English sentence. ".repeat(600)
21241            ),
21242        );
21243        runner.state.config.agents = vec![author];
21244        seed_public_summary(&mut runner);
21245        let winner = runner.state.candidates[0].clone();
21246        assert!(
21247            runner
21248                .guarded_pr_message(&winner, None, true)
21249                .await
21250                .unwrap()
21251                .is_none()
21252        );
21253        let record = runner.state.github_text.clone().unwrap();
21254        let rewrite = record.rewrite.clone().unwrap();
21255        assert_eq!(rewrite.failed, ["body-language"]);
21256        assert!(
21257            rewrite
21258                .body
21259                .ends_with(&format!("magi:run/{}", runner.state.id))
21260        );
21261        let store = ask::Questions::open();
21262        let q = store.get(record.question.as_ref().unwrap()).unwrap();
21263        assert!(
21264            q.choices
21265                .contains(&crate::github_text::USE_REWRITE.to_owned())
21266        );
21267        for label in [
21268            "ORIGINAL",
21269            "FALLBACK",
21270            "REWRITE",
21271            "Rules still failed: body-language",
21272            "truncated",
21273        ] {
21274            assert!(q.detail.contains(label), "{label}");
21275        }
21276        let artifact = std::fs::read_to_string(crate::run::artifact_path(
21277            &runner.state,
21278            &crate::github_text::artifact_name(&record.fingerprint),
21279        ))
21280        .unwrap();
21281        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
21282        // Restart from serialized state, then retry after a bad owner title.
21283        runner.state =
21284            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
21285        store
21286            .update(&q.id, |q| {
21287                q.say("まだ日本語")?;
21288                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
21289            })
21290            .unwrap();
21291        assert!(
21292            runner
21293                .guarded_pr_message(&winner, None, true)
21294                .await
21295                .unwrap()
21296                .is_none()
21297        );
21298        let retry = runner.state.github_text.as_ref().unwrap();
21299        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
21300        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
21301        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
21302        assert!(
21303            q.posting_comparison
21304                .as_ref()
21305                .unwrap()
21306                .matches_detail(&q.detail)
21307        );
21308        store
21309            .update(&q.id, |q| {
21310                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
21311            })
21312            .unwrap();
21313        let message = runner
21314            .guarded_pr_message(&winner, None, true)
21315            .await
21316            .unwrap()
21317            .unwrap();
21318        assert_eq!(message.title, rewrite.title);
21319        assert_eq!(message.body, rewrite.body);
21320        assert_eq!(
21321            runner.state.github_text.as_ref().unwrap().choice,
21322            Some(GithubTextChoice::Rewrite)
21323        );
21324        let again = runner
21325            .guarded_pr_message(&winner, None, true)
21326            .await
21327            .unwrap()
21328            .unwrap();
21329        assert_eq!(again.title, message.title);
21330        assert_eq!(again.body, message.body);
21331        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21332        assert!(
21333            !runner
21334                .state
21335                .events
21336                .iter()
21337                .any(|e| e.message.contains(&rewrite.title)
21338                    || e.message.contains("English sentence."))
21339        );
21340    }
21341
21342    #[tokio::test]
21343    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
21344        crate::run::pin_test_home();
21345        let dir = tempfile::tempdir().unwrap();
21346        let mut runner = runner_at(RunStatus::Gating);
21347        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
21348        runner.state.repo = dir.path().to_owned();
21349        runner.state.candidates[0].worktree = dir.path().to_owned();
21350        seed_public_summary(&mut runner);
21351        let winner = runner.state.candidates[0].clone();
21352        assert!(
21353            runner
21354                .guarded_pr_message(&winner, None, true)
21355                .await
21356                .unwrap()
21357                .is_none()
21358        );
21359        let record = runner.state.github_text.as_ref().unwrap();
21360        assert!(record.rewrite.is_none());
21361        // Records written before the additive field deserialize with no rewrite.
21362        let mut old = serde_json::to_value(record).unwrap();
21363        old.as_object_mut().unwrap().remove("rewrite");
21364        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
21365        assert!(old.rewrite.is_none());
21366        let id = record.question.clone().unwrap();
21367        ask::Questions::open()
21368            .update(&id, |q| {
21369                q.choices.push(crate::github_text::USE_REWRITE.into());
21370                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
21371            })
21372            .unwrap();
21373        let message = runner
21374            .guarded_pr_message(&winner, None, true)
21375            .await
21376            .unwrap()
21377            .unwrap();
21378        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
21379        assert_eq!(
21380            runner.state.github_text.as_ref().unwrap().choice,
21381            Some(GithubTextChoice::Fallback)
21382        );
21383    }
21384
21385    #[tokio::test]
21386    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
21387        crate::run::pin_test_home();
21388        let dir = tempfile::tempdir().unwrap();
21389        let mut runner = runner_at(RunStatus::Gating);
21390        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
21391        runner.state.repo = dir.path().to_owned();
21392        runner.state.candidates[0].worktree = dir.path().to_owned();
21393        seed_public_summary(&mut runner);
21394        let winner = runner.state.candidates[0].clone();
21395        assert!(
21396            runner
21397                .guarded_pr_message(&winner, None, true)
21398                .await
21399                .unwrap()
21400                .is_none()
21401        );
21402        let store = ask::Questions::open();
21403        let find = |run: &str| {
21404            store
21405                .list()
21406                .into_iter()
21407                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
21408                .unwrap()
21409        };
21410        let run = runner.state.id.clone();
21411        let q = find(&run);
21412        let fp = runner
21413            .state
21414            .github_text
21415            .as_ref()
21416            .unwrap()
21417            .fingerprint
21418            .clone();
21419        let name = crate::github_text::artifact_name(&fp);
21420        let path = crate::run::artifact_path(&runner.state, &name);
21421        assert!(q.detail.contains(&path.display().to_string()));
21422        assert!(q.detail.contains("title-language: title"));
21423        let saved = std::fs::read_to_string(&path).expect("artifact written");
21424        assert!(saved.contains("日本語のタイトル"));
21425        assert!(q.cwd.is_none());
21426        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
21427        // A bad replacement is refused and asked about once more.
21428        let say = |id: &str, text: &str, choice: &str| {
21429            store
21430                .update(id, |q| {
21431                    q.thread.push(ask::Turn {
21432                        who: ask::Who::Operator,
21433                        body: text.to_owned(),
21434                        at: jiff::Timestamp::now(),
21435                        note: None,
21436                    });
21437                    q.answer(ask::Answer::Choice(choice.to_owned()))
21438                })
21439                .unwrap();
21440        };
21441        say(&q.id, "まだ日本語", "use my text");
21442        assert!(
21443            runner
21444                .guarded_pr_message(&winner, None, true)
21445                .await
21446                .unwrap()
21447                .is_none()
21448        );
21449        let second = find(&run);
21450        assert_ne!(second.id, q.id);
21451        assert!(second.detail.contains("did not pass"));
21452        // The retry still compares the original text; the rejected
21453        // replacement is named by category only.
21454        assert!(second.detail.contains("日本語のタイトル"));
21455        assert!(!second.detail.contains("まだ日本語"));
21456        assert!(second.detail.contains(&path.display().to_string()));
21457        say(&second.id, "fix: retry failed requests", "use my text");
21458        let message = runner
21459            .guarded_pr_message(&winner, None, true)
21460            .await
21461            .unwrap()
21462            .expect("a vetted replacement is posted");
21463        assert_eq!(message.title, "fix: retry failed requests");
21464        assert_eq!(
21465            runner
21466                .state
21467                .github_text
21468                .as_ref()
21469                .unwrap()
21470                .chosen_title
21471                .as_deref(),
21472            Some("fix: retry failed requests")
21473        );
21474        assert!(runner.state.github_text.as_ref().unwrap().resolved);
21475        // Resumed again: the saved decision is applied, nothing is re-asked.
21476        let again = runner
21477            .guarded_pr_message(&winner, None, true)
21478            .await
21479            .unwrap()
21480            .unwrap();
21481        assert_eq!(again.title, message.title);
21482        assert_eq!(
21483            store
21484                .list()
21485                .into_iter()
21486                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
21487                .count(),
21488            2
21489        );
21490    }
21491
21492    #[test]
21493    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
21494        // What magi itself writes stays English under any configured language,
21495        // so a future localisation of these headings fails here. (The agents'
21496        // own text is also checked by the posting gate.)
21497        let mut state = state_with_summary(
21498            "add retries",
21499            "TITLE: fix(web): batch reads\n- reads run.json once",
21500        );
21501        state.config.graph.language = "ja".to_owned();
21502        let m = pr_message(&state, 'A');
21503        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
21504        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
21505
21506        // The task is the operator's own text: it goes in untouched, and the
21507        // fallback title (no summary) may be in its language too.
21508        let task = "今回やってほしいこと: results projector を直す";
21509        let mut state = state_with_summary(task, "- no title line");
21510        state.config.graph.language = "ja".to_owned();
21511        let m = pr_message(&state, 'A');
21512        assert_eq!(
21513            m.title,
21514            format!("chore: land candidate A of run {}", state.id)
21515        );
21516        assert!(
21517            m.body.contains(&format!(
21518                "<summary>Original task</summary>\n\n{task}\n\n</details>"
21519            )),
21520            "{}",
21521            m.body
21522        );
21523    }
21524
21525    #[test]
21526    fn pr_message_scrubs_home_paths_and_addresses() {
21527        let state = state_with_summary(
21528            "fix it in /Users/someone/src/x",
21529            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
21530        );
21531        let m = pr_message(&state, 'A');
21532        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
21533            assert!(!m.body.contains(leak), "{}", m.body);
21534        }
21535        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
21536    }
21537
21538    #[test]
21539    fn pr_message_survives_a_task_that_closes_details() {
21540        let state = state_with_summary("a </details> b", "TITLE: fix: x");
21541        let m = pr_message(&state, 'A');
21542        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
21543    }
21544
21545    #[test]
21546    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
21547        let cmd = manual_merge_command(
21548            MergeStyle::Squash,
21549            Path::new("/repo"),
21550            "b",
21551            "fix: \"quoted\" $(x) `y`\n\nbody",
21552        );
21553        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
21554    }
21555
21556    #[test]
21557    fn manual_merge_command_matches_the_configured_style() {
21558        let repo = Path::new("/repo");
21559        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
21560
21561        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
21562        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
21563
21564        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
21565        assert_eq!(
21566            squash,
21567            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
21568             \"Merge magi run 0832 (candidate A)\""
21569        );
21570
21571        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
21572        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
21573    }
21574
21575    #[test]
21576    fn a_nudge_gets_a_quarter_of_the_budget() {
21577        // The judge and implement budgets magi ships with.
21578        assert_eq!(retry_budget(secs(1200), true), secs(300));
21579        assert_eq!(retry_budget(secs(3600), true), secs(900));
21580    }
21581
21582    #[test]
21583    fn a_resent_prompt_keeps_the_whole_budget() {
21584        // The seat kept no context, so the retry is the original job again and
21585        // shortening it would only guarantee a second failure.
21586        assert_eq!(retry_budget(secs(1200), false), secs(1200));
21587        assert_eq!(retry_budget(secs(60), false), secs(60));
21588    }
21589
21590    #[test]
21591    fn the_floor_never_exceeds_the_original_budget() {
21592        // A short configured timeout must not be *raised* by the floor: the
21593        // operator asked for a bound, and a retry may not outlast the attempt
21594        // it is retrying.
21595        assert_eq!(retry_budget(secs(60), true), secs(60));
21596        assert_eq!(retry_budget(secs(480), true), secs(120));
21597        assert_eq!(retry_budget(secs(0), true), secs(0));
21598    }
21599
21600    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
21601        agent::CommandEvidence {
21602            id: "item1".to_owned(),
21603            description: "cargo test".to_owned(),
21604            exit_code,
21605            result_summary: String::new(),
21606            source: "codex".to_owned(),
21607        }
21608    }
21609
21610    #[test]
21611    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
21612        // No evidence is not the same fact as unconfirmed evidence: a
21613        // backend with no adapter, or a reply that ran no commands at all,
21614        // must not be misread as carrying a dangling job.
21615        assert!(!has_unconfirmed_command(&[]));
21616    }
21617
21618    #[test]
21619    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
21620        // Deliberately not a check on the exit code's *value*: a fixer
21621        // legitimately runs something that fails mid-iteration before it
21622        // succeeds, and that must never by itself reopen a valid report.
21623        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
21624        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
21625        assert!(!has_unconfirmed_command(&[
21626            evidence(Some(0)),
21627            evidence(Some(101))
21628        ]));
21629    }
21630
21631    #[test]
21632    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
21633        assert!(has_unconfirmed_command(&[
21634            evidence(Some(0)),
21635            evidence(None)
21636        ]));
21637    }
21638
21639    #[test]
21640    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
21641        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21642        assert_eq!(
21643            verified_noop_claim(true, &[], text).as_deref(),
21644            Some("already fixed by b32cfc4, on main.")
21645        );
21646    }
21647
21648    #[test]
21649    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
21650        // A timeout or a bad exit code reads as the ordinary loss it is,
21651        // whatever the reply's own prose claims.
21652        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21653        assert!(verified_noop_claim(false, &[], text).is_none());
21654    }
21655
21656    #[test]
21657    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
21658        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21659        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
21660        // A confirmed command alongside the marker is fine.
21661        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
21662    }
21663
21664    #[test]
21665    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
21666        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
21667    }
21668
21669    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
21670    /// pair, in order, labelled A, B, C, ...
21671    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
21672        runner.state.candidates = shape
21673            .iter()
21674            .enumerate()
21675            .map(|(i, &(empty, verified))| Candidate {
21676                index: i,
21677                label: (b'A' + i as u8) as char,
21678                agent: "sonnet".to_owned(),
21679                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
21680                worktree: PathBuf::from(format!("/wt/{i}")),
21681                summary: String::new(),
21682                stat: String::new(),
21683                files: 0,
21684                commits: 0,
21685                empty,
21686                failed: None,
21687                verified_noop: verified.map(str::to_owned),
21688                duration_ms: 0,
21689                folded: false,
21690            })
21691            .collect();
21692    }
21693
21694    #[test]
21695    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
21696        ask_test_home();
21697        let mut runner = runner_at(RunStatus::Implementing);
21698        set_candidates(
21699            &mut runner,
21700            &[
21701                (true, Some("already on main at b32cfc4")),
21702                (true, Some("same fix, see the existing test")),
21703            ],
21704        );
21705
21706        runner
21707            .after_implement()
21708            .expect("a verified no-op is not an error");
21709
21710        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
21711    }
21712
21713    #[test]
21714    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
21715        ask_test_home();
21716        let mut runner = runner_at(RunStatus::Implementing);
21717        // Candidate A declares a verified no-op; candidate B simply wrote
21718        // nothing and said nothing about why. One candidate's claim is not
21719        // the whole run's agreement.
21720        set_candidates(
21721            &mut runner,
21722            &[(true, Some("already on main at b32cfc4")), (true, None)],
21723        );
21724
21725        let err = runner
21726            .after_implement()
21727            .expect_err("an unverified empty candidate must still fail the run");
21728
21729        assert!(
21730            err.to_string().contains("no candidate produced a change"),
21731            "{err}"
21732        );
21733        assert_eq!(runner.state.status, RunStatus::Failed);
21734    }
21735
21736    #[test]
21737    fn after_implement_still_fails_an_ordinary_all_empty_run() {
21738        ask_test_home();
21739        let mut runner = runner_at(RunStatus::Implementing);
21740        set_candidates(&mut runner, &[(true, None), (true, None)]);
21741
21742        let err = runner
21743            .after_implement()
21744            .expect_err("no candidate declared anything; this is an ordinary failure");
21745
21746        assert!(
21747            err.to_string().contains("no candidate produced a change"),
21748            "{err}"
21749        );
21750        assert_eq!(runner.state.status, RunStatus::Failed);
21751    }
21752}
21753
21754/// A graft rollback whose restore could not be confirmed (see
21755/// [`Runner::graft`]).
21756#[derive(Debug)]
21757struct GraftRestoreFailed(String);
21758
21759impl std::fmt::Display for GraftRestoreFailed {
21760    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
21761        write!(
21762            f,
21763            "could not restore the winner to its pre-graft commit: {}",
21764            self.0
21765        )
21766    }
21767}
21768
21769impl std::error::Error for GraftRestoreFailed {}
21770
21771/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
21772/// not the agent: they never draw on an agent's own timeout, and running out of
21773/// them never reads as an agent failure.
21774#[cfg(test)]
21775mod resource_wait_tests {
21776    use super::*;
21777    use std::collections::BTreeMap;
21778
21779    fn sh(id: &str, script: &str) -> AgentSpec {
21780        AgentSpec {
21781            id: id.to_owned(),
21782            kind: crate::config::AgentKind::Command,
21783            model: None,
21784            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
21785            extra_args: Vec::new(),
21786            env: BTreeMap::new(),
21787            prompt_delivery: None,
21788            max_turn_secs: None,
21789        }
21790    }
21791
21792    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
21793        SeatJob {
21794            seat: SeatState::new("impl-A", &spec.id, 1),
21795            spec,
21796            cwd: dir.to_path_buf(),
21797            prompt: "do it".to_owned(),
21798            timeout,
21799            allow_write: true,
21800            sessions: false,
21801            artifacts: dir.join("art"),
21802            stem: "impl-A".to_owned(),
21803            handover: None,
21804        }
21805    }
21806
21807    fn state(resource_wait_secs: u64) -> RunState {
21808        let mut config = Config::default();
21809        config.graph.timeout_resource_wait = resource_wait_secs;
21810        RunState::new(
21811            PathBuf::from("/repo"),
21812            "main".to_owned(),
21813            "abc1234".to_owned(),
21814            "task".to_owned(),
21815            config,
21816        )
21817    }
21818
21819    fn hold(cache: &Path) -> crate::cache::Guard {
21820        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
21821        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
21822            crate::cache::AcquireOutcome::Acquired(g) => g,
21823            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
21824        }
21825    }
21826
21827    async fn run(
21828        jobs: Vec<SeatJob>,
21829        sem: Arc<Semaphore>,
21830        cache: Option<&Path>,
21831        st: &mut RunState,
21832    ) -> Vec<(usize, SeatState, AgentOutcome)> {
21833        let prompts = Prompts::default();
21834        let ctx = WaveCtx {
21835            run: "r",
21836            node: "implement",
21837            prompts: &prompts,
21838            cache,
21839            round: None,
21840            carry_seats: false,
21841            pauses: None,
21842        };
21843        wave(jobs, sem, &ctx, st, 0).await
21844    }
21845
21846    #[tokio::test]
21847    async fn verification_cache_wait_preserves_the_command_budget() {
21848        crate::run::pin_test_home();
21849        let tmp = tempfile::tempdir().unwrap();
21850        let cache = tmp.path().join("target");
21851        let guard = hold(&cache);
21852        let release = tokio::spawn(async move {
21853            tokio::time::sleep(Duration::from_millis(100)).await;
21854            guard.release();
21855        });
21856        let mut st = state(10);
21857        let budget = Duration::from_millis(50);
21858        let wait = repository_wait(&st, None);
21859        let (outcomes, _) = with_cache_lease(
21860            &mut st,
21861            Some(&cache),
21862            "gate",
21863            "gate",
21864            tmp.path(),
21865            "abc1234",
21866            budget,
21867            "test gate",
21868            &wait,
21869            |_, available| async move {
21870                assert_eq!(
21871                    available, budget,
21872                    "contention must not shorten verification"
21873                );
21874                (Vec::new(), false, Vec::new())
21875            },
21876        )
21877        .await
21878        .unwrap();
21879        release.await.unwrap();
21880        assert!(outcomes.is_empty(), "{outcomes:?}");
21881        assert!(
21882            st.events
21883                .iter()
21884                .any(|event| event.message.contains("waiting for"))
21885        );
21886        assert!(st.resource_wait.is_none());
21887        // The verification body no longer owns the lease after returning.
21888        hold(&cache).release();
21889    }
21890
21891    #[tokio::test]
21892    async fn verification_cache_wait_expires_without_starting_commands() {
21893        crate::run::pin_test_home();
21894        let tmp = tempfile::tempdir().unwrap();
21895        let cache = tmp.path().join("target");
21896        let _held = hold(&cache);
21897        let mut st = state(0);
21898        let wait = repository_wait(&st, None);
21899        let (outcomes, _) = with_cache_lease(
21900            &mut st,
21901            Some(&cache),
21902            "gate",
21903            "gate",
21904            tmp.path(),
21905            "abc1234",
21906            Duration::from_secs(60),
21907            "test gate",
21908            &wait,
21909            |_, _| async { panic!("commands must not start without the cache lease") },
21910        )
21911        .await
21912        .unwrap();
21913        assert_eq!(outcomes.len(), 1);
21914        assert!(outcomes[0].resource_blocked);
21915        assert!(st.resource_wait.is_none());
21916        assert!(matches!(
21917            st.resource_block.unwrap().cause,
21918            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
21919        ));
21920    }
21921
21922    #[tokio::test]
21923    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
21924        crate::run::pin_test_home();
21925        let tmp = tempfile::tempdir().unwrap();
21926        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
21927        let cache = tmp.path().join("cache");
21928        let mut state = state(21600);
21929        let wait = repository_wait(&state, None);
21930        let (outcomes, _) = with_cache_lease(
21931            &mut state,
21932            Some(&cache),
21933            "gate",
21934            "gate",
21935            tmp.path(),
21936            "head",
21937            Duration::from_secs(60),
21938            "gate",
21939            &wait,
21940            |_, _| async { panic!("freshness failure must not start verification") },
21941        )
21942        .await
21943        .unwrap();
21944        assert_eq!(outcomes.len(), 1);
21945        assert!(outcomes[0].resource_blocked);
21946        let block = state.resource_block.unwrap();
21947        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
21948        assert!(!block.description().contains("timed out"));
21949    }
21950
21951    #[tokio::test]
21952    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
21953        crate::run::pin_test_home();
21954        let tmp = tempfile::tempdir().unwrap();
21955        let cache = tmp.path().join("cache");
21956        let _held = hold(&cache);
21957        let mut state = state(21600);
21958        let pause = Pause::new();
21959        let interrupt = Pause::new();
21960        let cause = ParkCause::Operator;
21961        pause.park_for(cause.clone());
21962        let prompts = Prompts::default();
21963        let ctx = WaveCtx {
21964            run: "test",
21965            node: "fix",
21966            prompts: &prompts,
21967            cache: Some(&cache),
21968            round: None,
21969            carry_seats: false,
21970            pauses: Some((&pause, &interrupt)),
21971        };
21972        let result = wave(
21973            vec![job(
21974                sh("writer", "exit 99"),
21975                tmp.path(),
21976                Duration::from_secs(1),
21977            )],
21978            Arc::new(Semaphore::new(1)),
21979            &ctx,
21980            &mut state,
21981            0,
21982        )
21983        .await;
21984        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
21985        assert_eq!(result[0].1.turns, 0);
21986        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21987        assert_eq!(
21988            state.resource_block.unwrap().cause,
21989            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
21990        );
21991        assert!(state.active.is_empty());
21992    }
21993
21994    #[tokio::test]
21995    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
21996        crate::run::pin_test_home();
21997        let tmp = tempfile::tempdir().unwrap();
21998        let cache = tmp.path().join("target-a");
21999        let guard = hold(&cache);
22000        // Held for longer than the agent's whole 1s budget.
22001        tokio::spawn(async move {
22002            tokio::time::sleep(Duration::from_millis(1500)).await;
22003            guard.release();
22004        });
22005        let mut st = state(60);
22006        let results = run(
22007            vec![job(
22008                sh("alpha", "echo ok"),
22009                tmp.path(),
22010                Duration::from_secs(1),
22011            )],
22012            Arc::new(Semaphore::new(1)),
22013            Some(&cache),
22014            &mut st,
22015        )
22016        .await;
22017        assert!(
22018            matches!(results[0].2, AgentOutcome::Ok(_)),
22019            "the wait must not be charged to the agent"
22020        );
22021        assert!(st.resource_block.is_none());
22022        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
22023        assert!(st.resource_wait.is_none() && st.active.is_empty());
22024    }
22025
22026    #[tokio::test]
22027    #[cfg(unix)]
22028    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
22029        use std::os::fd::AsRawFd as _;
22030        crate::run::pin_test_home();
22031        let tmp = tempfile::tempdir().unwrap();
22032        crate::git::git(tmp.path(), &["init"]).await.unwrap();
22033        let common = tmp.path().join(".git");
22034        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
22035        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
22036        assert_eq!(
22037            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
22038            0
22039        );
22040        let mut st = state(1);
22041        st.config.coordination = crate::config::test_coordination();
22042        let mut reader = job(
22043            sh("reader", "echo must-not-run"),
22044            tmp.path(),
22045            Duration::from_secs(30),
22046        );
22047        reader.allow_write = false;
22048        let results = run(
22049            vec![reader.clone()],
22050            Arc::new(Semaphore::new(1)),
22051            None,
22052            &mut st,
22053        )
22054        .await;
22055        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
22056        assert_eq!(results[0].1.turns, 0);
22057        assert!(FailClass::of(&results[0].2).is_none());
22058        assert_eq!(
22059            st.resource_block.as_ref().unwrap().kind,
22060            "repository coordination"
22061        );
22062        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
22063        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
22064        drop(external);
22065        let editor = crate::repo_lock::SeatLease::for_checkout(
22066            tmp.path(),
22067            true,
22068            crate::config::test_coordination().as_ref(),
22069        )
22070        .await
22071        .unwrap();
22072        let pause = Pause::new();
22073        let interrupt = Pause::new();
22074        let prompts = Prompts::default();
22075        let ctx = WaveCtx {
22076            run: "r",
22077            node: "review",
22078            prompts: &prompts,
22079            cache: None,
22080            round: None,
22081            carry_seats: false,
22082            pauses: Some((&pause, &interrupt)),
22083        };
22084        let (results, ()) = tokio::join!(
22085            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
22086            async {
22087                tokio::time::sleep(Duration::from_millis(50)).await;
22088                pause.park();
22089            }
22090        );
22091        assert!(
22092            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
22093        );
22094        assert_eq!(results[0].1.turns, 0);
22095        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
22096        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
22097        drop(editor);
22098    }
22099
22100    #[tokio::test]
22101    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
22102        crate::run::pin_test_home();
22103        let tmp = tempfile::tempdir().unwrap();
22104        let cache = tmp.path().join("target-b");
22105        let _held = hold(&cache);
22106        let mut st = state(1);
22107        let results = run(
22108            vec![job(
22109                sh("alpha", "echo ok"),
22110                tmp.path(),
22111                Duration::from_secs(1),
22112            )],
22113            Arc::new(Semaphore::new(1)),
22114            Some(&cache),
22115            &mut st,
22116        )
22117        .await;
22118        let (_, seat, out) = &results[0];
22119        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
22120        assert!(wave_blocked(&results));
22121        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
22122        assert_eq!(seat.turns, 0, "nothing was asked");
22123        assert!(
22124            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
22125            "no agent was spawned"
22126        );
22127        assert!(FailClass::of(out).is_none(), "not a failure class");
22128        let block = st.resource_block.as_ref().expect("recorded for the resume");
22129        assert_eq!(block.kind, "shared build cache");
22130        assert!(matches!(
22131            block.cause,
22132            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
22133        ));
22134        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
22135        assert!(st.active.is_empty());
22136    }
22137
22138    #[tokio::test]
22139    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
22140        crate::run::pin_test_home();
22141        let tmp = tempfile::tempdir().unwrap();
22142        let mut st = state(1);
22143        let results = run(
22144            vec![job(
22145                sh("alpha", "echo ok"),
22146                tmp.path(),
22147                Duration::from_secs(1),
22148            )],
22149            Arc::new(Semaphore::new(0)),
22150            None,
22151            &mut st,
22152        )
22153        .await;
22154        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
22155        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
22156        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
22157    }
22158
22159    #[tokio::test]
22160    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
22161        crate::run::pin_test_home();
22162        let tmp = tempfile::tempdir().unwrap();
22163        let mut st = state(60);
22164        let results = run(
22165            vec![job(
22166                sh("alpha", "sleep 30"),
22167                tmp.path(),
22168                Duration::from_secs(1),
22169            )],
22170            Arc::new(Semaphore::new(1)),
22171            None,
22172            &mut st,
22173        )
22174        .await;
22175        let out = &results[0].2;
22176        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
22177        let class = FailClass::of(out).expect("a failure");
22178        assert_eq!(class, FailClass::Timeout);
22179        assert!(should_hand_over(None, &class), "existing handover policy");
22180        assert!(st.resource_block.is_none());
22181    }
22182}