Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
534/// on an early return or a panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615impl Runner {
616    /// Start a fresh run against `repo`.
617    pub async fn start(
618        repo: &Path,
619        instruction: String,
620        config: Config,
621        origin: Origin,
622    ) -> Result<Self> {
623        Self::start_naming(repo, instruction, "", config, origin).await
624    }
625
626    /// [`Runner::start`] for a queued task. Titles and instructions never
627    /// select implementation sources. The title argument is retained for callers.
628    pub async fn start_naming(
629        repo: &Path,
630        instruction: String,
631        _also_scan: &str,
632        config: Config,
633        origin: Origin,
634    ) -> Result<Self> {
635        let repo = git::toplevel(repo).await?;
636        let missing = agent::missing_programs(&config.agents);
637        if !missing.is_empty() {
638            bail!(
639                "these agent programs are not on PATH: {}. Fix the roster in \
640                 magi.toml or install them.",
641                missing.join(", ")
642            );
643        }
644        let base_branch =
645            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
646                .await?;
647        let fetched_base = resolve_base(
648            &repo,
649            &base_branch,
650            &config.merge.remote,
651            config.coordination.as_ref(),
652        )
653        .await?;
654        let base_commit = fetched_base.commit.clone();
655        let roles = config.resolve_roles()?;
656        let max_parallel = config.graph.max_parallel.max(1);
657        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
658        // Recorded before the first save, so a crash right after minting
659        // cannot leave a run with no origin. Legibility only: nothing reads it
660        // to decide anything.
661        state.fetched_base = Some(fetched_base);
662        state.origin = Some(origin);
663        state.reserve()?;
664        state.event("start", format!("run {} created", state.id));
665        state.save()?;
666        Ok(Self {
667            state,
668            roles,
669            sem: Arc::new(Semaphore::new(max_parallel)),
670            pause: Pause::new(),
671            interrupt: Pause::new(),
672        })
673    }
674
675    /// Open a review-only run against work that already exists on `branch`.
676    ///
677    /// The expensive half of the graph is the implement wave — measured at
678    /// 111 and 134 internal tool-loop turns on this repository, against a
679    /// handful for a judge or a reviewer. The cheap half is worth running on
680    /// hand-written work too, and there was no way to reach it.
681    ///
682    /// No new state and no schema change are needed: a run with **one** viable
683    /// candidate and a tally already decided degrades `execute` to exactly
684    /// review → gate → merge, because `judge` skips a single-candidate field,
685    /// `deliberate` has fewer than two first choices to reconcile, `vote`
686    /// returns early, `tally` is already present and `fold_losers` has no
687    /// losers. Resuming such a run therefore does the right thing as well.
688    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
689        Self::review_taking_over(repo, branch, config, None, origin).await
690    }
691
692    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
693    /// at the same task still has `branch` checked out, its worktree is
694    /// released first if that is safe (see [`crate::handover`]), and the
695    /// review refuses with the reason if it is not. `None` is a hand-run
696    /// review: it has no earlier attempts, so only a worktree of a dead run
697    /// magi recorded itself can be released.
698    pub async fn review_taking_over(
699        repo: &Path,
700        branch: &str,
701        config: Config,
702        takeover: Option<crate::handover::Takeover>,
703        origin: Origin,
704    ) -> Result<Self> {
705        let repo = git::toplevel(repo).await?;
706        let missing = agent::missing_programs(&config.agents);
707        if !missing.is_empty() {
708            bail!(
709                "these agent programs are not on PATH: {}. Fix the roster in \
710                 magi.toml or install them.",
711                missing.join(", ")
712            );
713        }
714        let base_branch =
715            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
716                .await?;
717        if base_branch == branch {
718            bail!("`{branch}` is the base branch; there is nothing to review against");
719        }
720        let fetched_base = resolve_base(
721            &repo,
722            &base_branch,
723            &config.merge.remote,
724            config.coordination.as_ref(),
725        )
726        .await?;
727        let base_commit = fetched_base.commit.clone();
728
729        let roles = config.resolve_roles()?;
730        let max_parallel = config.graph.max_parallel.max(1);
731        let mut state = RunState::new(
732            repo.clone(),
733            base_branch,
734            base_commit.clone(),
735            String::new(),
736            config,
737        );
738        state.fetched_base = Some(fetched_base);
739        state.origin = Some(origin);
740        state.reserve()?;
741
742        // Released before anything else touches the branch: a stale local
743        // branch is moved with `git branch -f`, which git refuses while an
744        // earlier attempt's worktree still has it checked out. Everything
745        // after this point that can fail puts the old run back.
746        // A hand-run review has no task, hence no earlier attempts, but a
747        // worktree of a dead run magi made may still be released.
748        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
749            earlier: Vec::new(),
750            home: crate::run::home(),
751            choice: None,
752            task_instruction: None,
753        });
754        let wait = repository_wait(&state, None);
755        let coordination_config = state.config.coordination.clone();
756        let coordination =
757            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
758                .await?;
759        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
760        if let Some(released) = &released {
761            state.event(
762                "release",
763                format!(
764                    "took `{branch}` over from run {}: its worktree was released: {}",
765                    crate::run::short_of(&released.old_id),
766                    released.audit
767                ),
768            );
769        }
770        // The owner's answer to an earlier divergence question is applied
771        // here: after the release (git will not move a checked-out branch)
772        // and before the sync that would otherwise ask again.
773        state.foreign_checkout = crate::handover::preflight_review(
774            &repo,
775            branch,
776            &takeover.home,
777            &state.config.merge.remote,
778        )
779        .await?;
780        if state.foreign_checkout.is_some() && takeover.choice.is_some() {
781            bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
782        }
783        if let Some(choice) = takeover.choice.as_ref()
784            && let Err(e) =
785                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
786                    .await
787        {
788            if let Some(released) = &released {
789                released.restore(&repo, branch).await;
790            }
791            return Err(e.context("applying the owner's answer about the diverged branch"));
792        }
793        // An earlier attempt of this task that published the task head, and
794        // whose work is the branch under review, hands that head over: the
795        // review run updates the same pull request. Without one (a legacy
796        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
797        // own head and its own open pull request is adopted as before.
798        for id in takeover.earlier.iter().rev() {
799            if let Ok(old) = RunState::load_under(id, &takeover.home)
800                && let Some(head) = old.pr_head.clone()
801                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
802            {
803                state.pr_head = Some(head);
804                break;
805            }
806        }
807        drop(coordination);
808        let opened = Self::open_review(
809            &repo,
810            branch,
811            state,
812            roles,
813            max_parallel,
814            base_commit,
815            &takeover,
816        )
817        .await;
818        if opened.is_err()
819            && let Some(released) = &released
820        {
821            let _coordination = crate::repo_lock::SeatLease::operation_with(
822                &repo,
823                coordination_config.as_ref(),
824                &wait,
825            )
826            .await?;
827            released.restore(&repo, branch).await;
828        }
829        opened
830    }
831
832    /// Recover exactly the operator-selected public implementation, without touching old sources.
833    pub async fn recover_pr(
834        repo: &Path,
835        task: &str,
836        selection: crate::pr_recovery::Selection,
837        mut config: Config,
838    ) -> Result<Self> {
839        selection.configure(&mut config)?;
840        selection.check_task(task, &selection.instruction)?;
841        let base_branch =
842            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
843                .await?;
844        let fetched_base = resolve_base(
845            repo,
846            &base_branch,
847            &config.merge.remote,
848            config.coordination.as_ref(),
849        )
850        .await?;
851        let base_commit = fetched_base.commit.clone();
852        selection
853            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
854            .await?;
855        Box::pin(selection.verify_source(repo)).await?;
856        if let Some(proof) = &selection.source_proof {
857            anyhow::ensure!(
858                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
859                "source rebase base is outside the fetched landing base"
860            );
861        }
862        let roles = config.resolve_roles()?;
863        let max_parallel = config.graph.max_parallel.max(1);
864        let mut state = RunState::new(
865            repo.to_path_buf(),
866            base_branch,
867            base_commit.clone(),
868            selection.instruction.clone(),
869            config,
870        );
871        state.fetched_base = Some(fetched_base);
872        state.origin = Some(crate::run::Origin::queue(task));
873        state.pr_head = Some(selection.branch.clone());
874        state.pr_recovery = Some(selection.clone());
875        state.reserve()?;
876        if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
877            // Never leave a reserved record behind as a phantom owner of the head.
878            state.discard_reserved();
879            return Err(e);
880        }
881        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
882    }
883
884    async fn open_selected(
885        repo: &Path,
886        state: RunState,
887        roles: ResolvedRoles,
888        max_parallel: usize,
889        base_commit: String,
890        selection: crate::pr_recovery::Selection,
891    ) -> Result<Self> {
892        let mut state = state;
893        let branch = state.branch_for('A');
894        let wait = repository_wait(&state, None);
895        let prepared = async {
896            let coordination = coordinate_operation(&mut state, repo, &wait).await?;
897            git::git(repo, &["branch", &branch, selection.source()]).await?;
898            drop(coordination);
899            anyhow::Ok(())
900        }
901        .await;
902        if let Err(e) = prepared {
903            // Nothing was exposed yet: do not leave a reserved record as a
904            // phantom owner of the public head.
905            state.discard_reserved();
906            return Err(e);
907        }
908        state.event(
909            "recovery",
910            format!(
911                "selected implementation {} for {} (public lease {})",
912                selection.source(),
913                selection.url,
914                selection.head
915            ),
916        );
917        let takeover = crate::handover::Takeover {
918            earlier: Vec::new(),
919            home: crate::run::home(),
920            choice: None,
921            task_instruction: Some(selection.instruction.clone()),
922        };
923        if let Err(e) = state.save() {
924            state.discard_reserved();
925            return Err(e);
926        }
927        let id = state.id.clone();
928        let opened = Self::open_review(
929            repo,
930            &branch,
931            state,
932            roles,
933            max_parallel,
934            base_commit,
935            &takeover,
936        )
937        .await;
938        match opened {
939            Ok(runner) => Ok(runner),
940            Err(e) => {
941                if let Ok(mut partial) = RunState::load(&id) {
942                    partial.status = RunStatus::Blocked;
943                    partial.driver_exited = true;
944                    partial.event(
945                        "recovery",
946                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
947                    );
948                    partial.save()?;
949                }
950                Err(crate::pr_recovery::StartupFailed {
951                    run: id,
952                    cause: format!("{e:#}"),
953                }
954                .into())
955            }
956        }
957    }
958
959    /// What the newest earlier attempt that reviewed `branch` leaves for this
960    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
961    /// no earlier attempts. Reads the earlier record only: ownership of its
962    /// worktree and branch is [`crate::handover`]'s business.
963    async fn inherit(
964        repo: &Path,
965        branch: &str,
966        base_commit: &str,
967        takeover: &crate::handover::Takeover,
968    ) -> Option<crate::recovery::RecoveryRecord> {
969        use crate::recovery::{Relation, assess};
970        let newest = takeover.earlier.last()?;
971        let mut chosen = None;
972        for id in takeover.earlier.iter().rev() {
973            if let Ok(old) = RunState::load_under(id, &takeover.home)
974                && !old.reviews.is_empty()
975                && (old.winner().is_some_and(|w| w.branch == branch)
976                    || old.pr_head.as_deref() == Some(branch))
977            {
978                chosen = Some(old);
979                break;
980            }
981        }
982        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
983            .await
984            .unwrap_or_default();
985        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
986            Some(last) if last.head == tip => Relation::Same,
987            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
988            _ => Relation::Diverged,
989        };
990        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
991        Some(assess(from, chosen.as_ref(), base_commit, relation))
992    }
993
994    /// The half of [`Runner::review_taking_over`] that can fail after an
995    /// earlier attempt's worktree was released.
996    async fn open_review(
997        repo: &Path,
998        branch: &str,
999        mut state: RunState,
1000        roles: ResolvedRoles,
1001        max_parallel: usize,
1002        base_commit: String,
1003        takeover: &crate::handover::Takeover,
1004    ) -> Result<Self> {
1005        let wait = repository_wait(&state, None);
1006        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1007        if state.foreign_checkout.is_some() {
1008            // Never reconcile or force-move a ref checked out by the operator.
1009            if state.pr_head.is_none() {
1010                state.pr_head = Some(branch.to_owned());
1011            }
1012        } else if state.pr_recovery.is_none() {
1013            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1014        }
1015        // The commit subjects are the closest thing to a task statement that
1016        // existing work carries, and the reviewers are told as much.
1017        let start = review_base(
1018            repo,
1019            &state.config.merge.remote,
1020            &state.base_branch,
1021            &base_commit,
1022            branch,
1023        )
1024        .await;
1025        let log = git::log_oneline(repo, &start, branch)
1026            .await
1027            .unwrap_or_default();
1028        let log_text = if log.trim().is_empty() {
1029            "(no commit messages)"
1030        } else {
1031            log.trim()
1032        };
1033        let mut instruction = format!(
1034            "Review the work already on branch `{branch}`. There is no task \
1035             statement: what the change claims to do is whatever its commits \
1036             say.\n\n{log_text}"
1037        );
1038        // A retry of a task whose earlier attempt left reviews behind carries
1039        // that attempt's evidence, and the task's own statement with it: the
1040        // completion criteria are the task's, not whatever the commits claim.
1041        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1042            if let Some(task) = takeover
1043                .task_instruction
1044                .as_deref()
1045                .filter(|t| !t.trim().is_empty())
1046                && rec.reviewed_head.is_some()
1047            {
1048                instruction = format!(
1049                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1050                    task.trim()
1051                );
1052                rec.criteria_inherited = true;
1053            }
1054            state.event("recovery", rec.summary());
1055            state.recovery = Some(rec);
1056        }
1057        if let Some(selected) = &state.pr_recovery {
1058            instruction = selected.instruction.clone();
1059        }
1060        state.task_instruction = takeover.task_instruction.clone();
1061        state.instruction = instruction;
1062        state.reviewed_commits = Some(
1063            git::subjects(repo, &start, branch)
1064                .await
1065                .unwrap_or_default(),
1066        );
1067
1068        // A takeover must not forget a hold that a previous review observed,
1069        // even if repairs have since removed it from the branch.
1070        for id in &takeover.earlier {
1071            if let Ok(old) = RunState::load_under(id, &takeover.home)
1072                && old.repo == state.repo
1073                && (old.pr_head.as_deref() == Some(branch)
1074                    || old.winner().is_some_and(|w| w.branch == branch))
1075            {
1076                for hold in old.branch_owner_holds {
1077                    if !state.branch_owner_holds.contains(&hold) {
1078                        state.event(
1079                            "approval-policy",
1080                            format!(
1081                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1082                                hold.control.key(),
1083                                hold.file,
1084                                hold.head
1085                            ),
1086                        );
1087                        state.branch_owner_holds.push(hold);
1088                    }
1089                }
1090            }
1091        }
1092        // Select the actual review tip before observing owner policy. A foreign
1093        // checkout can lag its remote descendant, and repairs must not erase
1094        // holds introduced only on that selected commit.
1095        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1096            let tracking = format!("{}/{branch}", state.config.merge.remote);
1097            // A surviving tracking ref is not evidence that the remote still
1098            // offers this tip. Adopt it only after a successful fetch here.
1099            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1100            let mut head = checkout.synced_head.clone();
1101            if matches!(fetched, Ok(out) if out.ok()) {
1102                let remote = git::rev_parse(repo, &tracking).await?;
1103                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1104                    head = remote;
1105                }
1106            }
1107            head
1108        } else {
1109            git::rev_parse(repo, branch).await?
1110        };
1111        state.observe_branch_owner_holds(&reviewed_head).await;
1112
1113        // An attached worktree, so the fixer's commits land on the branch under
1114        // review rather than on a detached head nobody will look at again.
1115        let worktree = state.worktree_root().join("under-review");
1116        if let Some(parent) = worktree.parent() {
1117            tokio::fs::create_dir_all(parent).await.ok();
1118        }
1119        let path = worktree.to_string_lossy().to_string();
1120        let review_branch = if state.foreign_checkout.is_some() {
1121            let shadow = format!("magi/{}/review", state.short());
1122            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1123            shadow
1124        } else {
1125            branch.to_owned()
1126        };
1127        git::git(repo, &["worktree", "add", &path, &review_branch])
1128            .await
1129            .with_context(|| {
1130                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1131            })?;
1132
1133        let setup = match ensure_setup_config(&mut state, repo).await {
1134            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1135            Err(e) => Err(e),
1136        };
1137        if let Err(e) = setup {
1138            git::worktree_remove_clean(repo, &worktree).await.ok();
1139            return Err(e);
1140        }
1141        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1142            .await
1143            .unwrap_or(0);
1144        if commits == 0 {
1145            git::worktree_remove_clean(repo, &worktree).await.ok();
1146            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1147        }
1148        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1149            .await
1150            .map(|f| f.len())
1151            .unwrap_or(0);
1152        if files == 0
1153            && let (Ok(head_tree), Ok(base_tree)) = (
1154                git::tree_of(&worktree, "HEAD").await,
1155                git::tree_of(&worktree, &base_commit).await,
1156            )
1157            && head_tree == base_tree
1158        {
1159            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1160            git::worktree_remove_clean(repo, &worktree).await.ok();
1161            bail!(
1162                "`{branch}` at {} has a tree identical to base {}; this usually means \
1163                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1164                 against `{}/{branch}`) rather than an empty change",
1165                short(&head),
1166                short(&base_commit),
1167                state.config.merge.remote
1168            );
1169        }
1170        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1171            .await
1172            .unwrap_or_default();
1173
1174        state.candidates.push(Candidate {
1175            index: 0,
1176            label: 'A',
1177            // Not an agent id on purpose: nothing in the roster wrote this, and
1178            // the stats tables must not credit anyone with a win for it.
1179            agent: EXISTING_BRANCH.to_owned(),
1180            branch: review_branch,
1181            worktree,
1182            summary: String::new(),
1183            stat,
1184            files,
1185            commits,
1186            empty: false,
1187            failed: None,
1188            verified_noop: None,
1189            duration_ms: 0,
1190            folded: false,
1191        });
1192        state.tally = Some(Tally {
1193            first_choice: BTreeMap::from([('A', 0)]),
1194            borda: BTreeMap::new(),
1195            winner: 'A',
1196            rankings: 0,
1197            unanimous_initial: false,
1198            deliberated: false,
1199            changed_votes: 0,
1200            unanimous_final: false,
1201            tie_break: None,
1202            // No panel sat, so no quorum applies. Zero judges is the correct
1203            // number for work that never competed, and must not be reported as
1204            // a collapsed panel.
1205            judges: 0,
1206            present: 0,
1207            quorum: 0,
1208            met_quorum: true,
1209            uncontested: Some("review-only run: nothing competed".to_owned()),
1210        });
1211        state.status = RunStatus::Reviewing;
1212        state.prep_complete = Some(true);
1213        state.event(
1214            "start",
1215            format!(
1216                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1217                state.id
1218            ),
1219        );
1220        state.save()?;
1221        Ok(Self {
1222            state,
1223            roles,
1224            sem: Arc::new(Semaphore::new(max_parallel)),
1225            pause: Pause::new(),
1226            interrupt: Pause::new(),
1227        })
1228    }
1229
1230    /// Authorize exactly one additional rebase, under the caller's queue task
1231    /// claim and the same owner/execution locks used by run maintenance.
1232    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1233        Self::authorize_base_recovery_request(home, id, task, false).await
1234    }
1235
1236    /// Queue release uses its durable approval generation exactly once.
1237    pub(crate) async fn authorize_base_recovery_from_resumption(
1238        home: &Path,
1239        id: &str,
1240        task: &str,
1241    ) -> Result<()> {
1242        Self::authorize_base_recovery_request(home, id, task, true).await
1243    }
1244
1245    async fn authorize_base_recovery_request(
1246        home: &Path,
1247        id: &str,
1248        task: &str,
1249        resumption: bool,
1250    ) -> Result<()> {
1251        let _owner = crate::pr_head::owner_lock(home, id)?;
1252        let mut state = RunState::load_under(id, home)?;
1253        crate::pr_head::ensure_owner(&state)?;
1254        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1255        anyhow::ensure!(
1256            queued.runs.last().is_some_and(|run| run == id)
1257                && queued.status != crate::queue::TaskStatus::Done
1258                && !queued.fresh_start
1259                && queued.review_branch.is_none(),
1260            "run task ownership changed"
1261        );
1262        anyhow::ensure!(
1263            state
1264                .origin
1265                .as_ref()
1266                .and_then(|o| o.task.as_deref())
1267                .is_none_or(|owner| owner == task),
1268            "run belongs to another task"
1269        );
1270        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1271        anyhow::ensure!(
1272            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1273            "run is live or its ownership is unknown"
1274        );
1275        // Judge the previous driver's lease before acquiring our maintenance
1276        // lease. The owner lock excludes new driver claims across both steps.
1277        let _execution = crate::driver::maintenance(home, id)?;
1278        anyhow::ensure!(
1279            state.status.resumable()
1280                && state.released_to.is_none()
1281                && state.retired_to.is_none()
1282                && state.pr_taken_over_by.is_none(),
1283            "run ownership is final"
1284        );
1285        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1286        anyhow::ensure!(
1287            !state.pr_publication_pending
1288                && !crate::handover::landing_checkpoint(&state)
1289                && state.merge_decision.is_none(),
1290            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1291        );
1292        let winner = state.winner().context("run has no retained winner")?;
1293        anyhow::ensure!(
1294            git::is_clean(&winner.worktree).await?,
1295            "retained worktree has uncommitted changes"
1296        );
1297        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1298        anyhow::ensure!(
1299            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1300            "retained worktree does not match the winner branch"
1301        );
1302        state.record_base_sync_stop(&head);
1303        state.save()?;
1304        if resumption {
1305            anyhow::ensure!(
1306                crate::daemon::compatible_run_instruction(&state, &queued),
1307                "resumption instruction does not match the retained run"
1308            );
1309            let approval = queued
1310                .resumption
1311                .as_ref()
1312                .context("no operator resumption")?;
1313            anyhow::ensure!(
1314                approval.instruction == queued.instruction,
1315                "resumption approval does not match the current task instruction"
1316            );
1317            state.authorize_base_sync_resumption(task, approval)?;
1318        } else {
1319            state.authorize_base_sync(task)?;
1320        }
1321        state.event(
1322            "base-recovery",
1323            "owner authorized one additional base rebase; fresh verification required",
1324        );
1325        state.save()?;
1326        Ok(())
1327    }
1328
1329    /// Reopen an existing run.
1330    pub fn resume(id: &str) -> Result<Self> {
1331        let state = RunState::load(id)?;
1332        crate::pr_head::ensure_owner(&state)?;
1333        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1334            bail!(
1335                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1336            );
1337        }
1338        if let Some(to) = &state.released_to {
1339            bail!(
1340                "run {} cannot be resumed: its worktree was released to run {}",
1341                state.short(),
1342                crate::run::short_of(to)
1343            );
1344        }
1345        let roles = state.config.resolve_roles()?;
1346        if let Some(id) = &state.config.graph.selected_implementer
1347            && state.candidates.iter().any(|c| c.agent != *id)
1348        {
1349            bail!(
1350                "replacement implementer pin does not match the saved candidates; task must stay held"
1351            );
1352        }
1353        let max_parallel = state.config.graph.max_parallel.max(1);
1354        Ok(Self {
1355            state,
1356            roles,
1357            sem: Arc::new(Semaphore::new(max_parallel)),
1358            pause: Pause::new(),
1359            interrupt: Pause::new(),
1360        })
1361    }
1362
1363    /// Walk the graph to a terminal state, skipping nodes already recorded.
1364    ///
1365    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1366    /// the phone - ends here, so this is the one place a run that ended
1367    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1368    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1369    pub async fn execute(&mut self) -> Result<()> {
1370        if let Some(selection) = &self.state.pr_recovery
1371            && (selection.source_proof.is_some() || selection.source_run.is_some())
1372        {
1373            Box::pin(selection.verify_source(&self.state.repo)).await?;
1374        }
1375        // Queue claims cover daemon work; this also serializes manual drivers
1376        // consuming a repair answer, including the new review and decision.
1377        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1378            || self.state.land_approval.is_some()
1379            || self.state.merge_decision.is_some()
1380            || !self.state.merge_repairs.is_empty()
1381            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1382        {
1383            let claim = FixClaim::acquire(&self.state.dir())?;
1384            let disk = RunState::load(&self.state.id)?;
1385            let checkpoint = |s: &RunState| {
1386                (
1387                    s.merge_repairs.len(),
1388                    s.merge_repairs
1389                        .last()
1390                        .map(|r| (r.question.clone(), r.phase)),
1391                    s.land_approval
1392                        .as_ref()
1393                        .map(|a| (a.question.clone(), a.head.clone())),
1394                )
1395            };
1396            if checkpoint(&disk) != checkpoint(&self.state) {
1397                bail!(
1398                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1399                );
1400            }
1401            Some(claim)
1402        } else {
1403            None
1404        };
1405        let mut execution = self.claim_driver().await?;
1406        let runner = &mut *execution.runner;
1407        let wait = runner.repository_wait();
1408        // Keep the graph's large future off the callers' stack, including
1409        // standalone task attempts running on a default-sized test thread.
1410        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1411        let result = match result {
1412            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1413                record_repository_block(&mut runner.state, &e);
1414                runner.park_here()?;
1415                Ok(())
1416            }
1417            other => other,
1418        };
1419        execution.driver.finish(&mut runner.state).await?;
1420        let ended = if result.is_err() {
1421            Some({
1422                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1423                if let Err(error) = &result {
1424                    notice.message = format!(
1425                        "Run {} stopped with an error: {}",
1426                        runner.state.id,
1427                        error.to_string().lines().next().unwrap_or("unknown error")
1428                    );
1429                }
1430                notice
1431            })
1432        } else {
1433            crate::notices::run_ended(&runner.state)
1434        };
1435        if let Some(notice) = ended {
1436            crate::notices::raise(notice);
1437        }
1438        result
1439    }
1440
1441    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1442        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1443    }
1444
1445    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1446        let wait = self.repository_wait();
1447        coordinate_operation(&mut self.state, cwd, &wait).await
1448    }
1449
1450    /// Validate ownership and persist the driver claim in one critical section.
1451    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1452        let owner = match crate::pr_head::owner_lock_wait(
1453            &crate::run::home(),
1454            &self.state.id,
1455            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1456        )
1457        .await
1458        {
1459            Ok(owner) => owner,
1460            Err(error) => {
1461                let detail = format!("worktree maintenance or ownership update: {error:#}");
1462                // Admission did not own run.json: retain this resource outcome
1463                // in memory for queue settlement without overwriting a concurrent
1464                // retirement or publication transaction.
1465                self.state.resource_block = Some(crate::run::ResourceBlock {
1466                    cause: crate::run::ResourceBlockCause::Other,
1467                    node: "start".into(),
1468                    kind: "worktree maintenance".into(),
1469                    detail: detail.clone(),
1470                    at: Timestamp::now(),
1471                });
1472                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1473            }
1474        };
1475        let disk = RunState::load(&self.state.id)?;
1476        crate::pr_head::ensure_owner(&disk)?;
1477        let claimed =
1478            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1479        // Only proven-live ownership refuses here. Unknown records (legacy runs
1480        // with no claim marker or process identity) fall through to the
1481        // exclusive execution lease taken just below, which is what actually
1482        // keeps a second driver out.
1483        anyhow::ensure!(
1484            disk.liveness(claimed) != crate::run::Liveness::Live,
1485            "run already has a live driver; wait for its execution to end"
1486        );
1487        if let Some(to) = &disk.released_to {
1488            bail!(
1489                "run {} cannot continue: its worktree was released to run {}",
1490                self.state.short(),
1491                crate::run::short_of(to)
1492            );
1493        }
1494        anyhow::ensure!(
1495            serde_json::to_value(&disk.base_sync_recovery)?
1496                == serde_json::to_value(&self.state.base_sync_recovery)?,
1497            "base recovery changed since this run was loaded; reload the saved run"
1498        );
1499        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1500        let mut execution = DriverExecution {
1501            runner: self,
1502            driver,
1503        };
1504        // Declare after the execution guard: on any claim error this short
1505        // lock must drop before that guard reloads and releases ownership.
1506        let _owner = owner;
1507        let state = &mut execution.runner.state;
1508        // Moving again, so it is no longer parked. Set before the walk rather
1509        // than in `resume`, so every way of re-entering the graph clears it
1510        // and a card cannot claim a run is waiting to be resumed while the
1511        // agents are already working.
1512        state.parked = false;
1513        state.resource_wait = None;
1514        state.resource_block = None;
1515        // Any seat this state still lists as answering belongs to whatever
1516        // process last drove this run — this one included, if it crashed
1517        // mid-wave. Cleared and flushed immediately, before anything else
1518        // runs, so a resume can never show a seat as live when nothing is
1519        // asking it anything yet; the node that actually dispatches the next
1520        // wave repopulates it.
1521        state.clear_active();
1522        // Recorded in the same spot, and flushed together with the clear
1523        // above: this is the pid a reader checks (`RunState::liveness`) when
1524        // no daemon claim exists to answer "is a process still driving this
1525        // run" — a plain `magi run` / `magi review` typed into a terminal
1526        // claims nothing there. Always overwritten, never only-if-absent, so
1527        // a resumed run's stale pid from a previous, possibly-dead process
1528        // can never survive into this one's own report. Unlike
1529        // `clear_active`, this changes on every single `execute()` call, so
1530        // the save below is now unconditional rather than only-if-cleared.
1531        //
1532        // `driver_started_at` is recorded in the same breath, from this same
1533        // pid, so `liveness` can tell a live pid that is genuinely still us
1534        // apart from one the OS has since handed to an unrelated process —
1535        // see that field's own doc for why the pid alone is not enough.
1536        let pid = std::process::id();
1537        state.driver_pid = Some(pid);
1538        state.driver_started_at = crate::proc::process_started_at(pid);
1539        state.driver_exited = false;
1540        state.driver_claimed = Some(true);
1541        state.begin_pass();
1542        execution.driver.bind(state);
1543        state.save()?;
1544        execution.driver.record(state)?;
1545        Ok(execution)
1546    }
1547
1548    async fn execute_graph(&mut self) -> Result<()> {
1549        self.state.configure_recovery_resume()?;
1550        // Preservation may invalidate an old empty classification. Resolve it
1551        // before any resume shortcut can reuse a verdict or publish a branch.
1552        self.persist_candidates().await?;
1553        // A saved base stop outranks stage markers, including legacy Voting.
1554        // Recovery never reenters implementation, deliberation or voting.
1555        if self.state.base_sync_blocked() {
1556            self.sync_to_base().await?;
1557            if !self.state.base_sync_blocked()
1558                && self.state.status != RunStatus::AlreadyInBase
1559                && !self.resource_halt()?
1560            {
1561                self.finish_after_tally().await?;
1562            }
1563            return Ok(());
1564        }
1565        if self
1566            .state
1567            .merge_decision
1568            .as_ref()
1569            .is_some_and(|decision| !decision.closed)
1570            && self.state.status != RunStatus::Landing
1571        {
1572            if crate::merge_decision::resumable_hold(&self.state) {
1573                self.state.merge = Some(MergeOutcome {
1574                    mode: MergeMode::Pr,
1575                    ok: false,
1576                    detail: self
1577                        .state
1578                        .merge_decision
1579                        .as_ref()
1580                        .unwrap()
1581                        .target
1582                        .pr
1583                        .clone(),
1584                    empty: false,
1585                });
1586                self.run_land().await?;
1587                self.settle_questions();
1588                return Ok(());
1589            }
1590            bail!(
1591                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1592            );
1593        }
1594        if self.state.config.graph.selected_implementer.is_some()
1595            && self.state.status == RunStatus::Blocked
1596            && self.state.tally.is_none()
1597            && self.state.candidate_preservation.is_empty()
1598            && self.state.interrupted_implementations.is_empty()
1599        {
1600            bail!(
1601                "selected implementation is held; explicitly release the owning task to retry it"
1602            );
1603        }
1604
1605        if self.state.merge_repairs.last().is_some_and(|r| {
1606            matches!(
1607                r.phase,
1608                crate::run::MergeRepairPhase::Requested
1609                    | crate::run::MergeRepairPhase::Fixing
1610                    | crate::run::MergeRepairPhase::Reviewing
1611            )
1612        }) {
1613            if self.repair_merge_review().await? {
1614                self.run_land().await?;
1615            }
1616            self.settle_questions();
1617            return Ok(());
1618        }
1619        // Publication already passed review and gate. Resume only that node:
1620        // syncing to a newer base here could change the verified commit.
1621        if self.state.pr_publication_pending {
1622            self.merge().await?;
1623            if self.resource_halt()? {
1624                return Ok(());
1625            }
1626            self.state.save()?;
1627            self.settle_questions();
1628            return Ok(());
1629        }
1630        // A run that already lost its quorum never resumes into the verdict
1631        // machinery: `deliberate` and `vote` would otherwise clobber the
1632        // stalled marker back to Voting and the run would keep going past a
1633        // verdict that is no longer trustworthy. Everything already recorded is
1634        // kept, so the run stays resumable (or foldable) for a human to pick up.
1635        //
1636        // On --resume the run gets one chance to repair itself: the seats a
1637        // rate limit took out are re-asked. If their quota has since reset and
1638        // the quorum is restored, the run picks up and finishes; otherwise it
1639        // stays stale and still-resumable for a later retry. If it does not
1640        // recover, the returned status stays `Stalled` and nothing was
1641        // clobbered (the recovery only mutates entries for the lost seats).
1642        if self.state.status == RunStatus::Stalled {
1643            if self.recover_stall().await? {
1644                self.finish_after_tally().await?;
1645            } else {
1646                // Still below quorum: persist the marker and stay resumable.
1647                self.state.save()?;
1648            }
1649            return Ok(());
1650        }
1651        // A run parked inside `land` - watching CI, mid fix-round, or
1652        // waiting on the owner's merge approval - resumes directly into it,
1653        // never back through `prep`. Everything before `merge` already
1654        // concluded; that is the only way `status` reaches `Landing` in the
1655        // first place. Re-walking `review_loop` first would also be actively
1656        // wrong: its own status recomputation (see its doc) treats any
1657        // clean round as reason to set `status` to `Gating`, which would
1658        // clobber this marker before `merge` ever ran, and this run would
1659        // never find its way back into `land` at all.
1660        if self.state.status == RunStatus::Landing
1661            || crate::handover::landing_checkpoint(&self.state)
1662        {
1663            self.state.land_sync_pending = true;
1664            self.state.save()?;
1665            self.run_land().await?;
1666            // `run_land` may have settled the run right here - CI came back
1667            // green and the PR merged, say - without ever passing back
1668            // through `merge`'s own trailing call. Whatever it left `status`
1669            // as is what this has to read.
1670            self.settle_questions();
1671            return Ok(());
1672        }
1673        // A run parked at `merge` on the owner's word about a withheld PR text
1674        // resumes straight into `merge`. Re-walking the tail would sync to the
1675        // base again (possibly rebasing, possibly spending the fixer) after the
1676        // gate had passed, and then push a tree nobody reviewed or gated.
1677        if self.state.github_text.is_some()
1678            && self.state.merge.is_none()
1679            && self.state.status == RunStatus::Gating
1680        {
1681            self.merge().await?;
1682            if self.resource_halt()? {
1683                return Ok(());
1684            }
1685            self.state.save()?;
1686            return Ok(());
1687        }
1688        if let Err(e) = self.prep().await {
1689            if e.is::<crate::repo_lock::ResourceBlocked>() {
1690                return Err(e);
1691            }
1692            self.state.status = RunStatus::Blocked;
1693            self.state
1694                .prep_failure
1695                .get_or_insert_with(|| format!("{e:#}"));
1696            self.state
1697                .event("prep", format!("preparation stopped: {e:#}"));
1698            self.state.save()?;
1699            return Err(e);
1700        }
1701        if self.park_here()? {
1702            return Ok(());
1703        }
1704        self.advise().await?;
1705        if self.park_here()? {
1706            return Ok(());
1707        }
1708        self.implement().await?;
1709        if self.park_here()? {
1710            return Ok(());
1711        }
1712        // `after_implement` already saved the state and settled any open
1713        // questions when it set this; nothing later in the graph has
1714        // anything to judge.
1715        if self.state.status == RunStatus::VerifiedNoop {
1716            return Ok(());
1717        }
1718        self.judge().await?;
1719        if self.park_here()? {
1720            return Ok(());
1721        }
1722        self.deliberate().await?;
1723        if self.park_here()? {
1724            return Ok(());
1725        }
1726        self.vote().await?;
1727        if self.park_here()? {
1728            return Ok(());
1729        }
1730        self.tally()?;
1731        // A verdict that lost its quorum is not trustworthy: do not review,
1732        // gate, or merge on it. Everything already done is kept, so the run
1733        // stays resumable (or foldable); the human can replace the agent that
1734        // ran out of quota and pick it up.
1735        if self.state.status == RunStatus::Stalled {
1736            // Persist the stalled marker now — the normal end-of-execute save
1737            // below is below this early return, and without it a resumed run
1738            // would reload a pre-tally status and keep going.
1739            self.state.save()?;
1740            return Ok(());
1741        }
1742        self.finish_after_tally().await?;
1743        Ok(())
1744    }
1745
1746    /// A wave could not start its agents because a shared resource stayed busy
1747    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1748    /// resumable. Unstarted seats have no result; completed peers retain
1749    /// their saved candidate state.
1750    fn stop_for_resource(&mut self) -> Result<()> {
1751        // Resource acquisition observes cancellation inside a node. Complete
1752        // the same durable park as every other node boundary before returning.
1753        if self.pause.parked() || self.interrupt.parked() {
1754            self.park_here()?;
1755            return Ok(());
1756        }
1757        if self.state.resource_block.is_some() {
1758            self.state.status = RunStatus::Blocked;
1759        }
1760        self.state.save()
1761    }
1762
1763    /// Stop walking the graph if a node ended on a resource block.
1764    fn resource_halt(&mut self) -> Result<bool> {
1765        if self.state.resource_block.is_none() {
1766            return Ok(false);
1767        }
1768        self.stop_for_resource()?;
1769        Ok(true)
1770    }
1771
1772    /// Park here if asked to, recording it in the run's own timeline.
1773    ///
1774    /// Returns whether the caller should stop walking the graph. The state is
1775    /// saved either way by the node that just finished; this adds the event so
1776    /// the operator's card says why a run that is neither finished nor moving
1777    /// is sitting where it is.
1778    fn park_here(&mut self) -> Result<bool> {
1779        let parking = self.pause.parked() || self.interrupt.parked();
1780        if !parking && self.state.resource_block.is_some() {
1781            self.stop_for_resource()?;
1782            return Ok(true);
1783        }
1784        // Either handle asking is enough - see `Pause`'s own doc for why
1785        // they are never the same one. `interrupt` is checked second so a
1786        // reason it carries is preferred in the message below over a plain
1787        // shutdown park racing it at the same boundary.
1788        if !self.pause.parked() && !self.interrupt.parked() {
1789            return Ok(false);
1790        }
1791        if self.state.parked {
1792            return Ok(true);
1793        }
1794        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1795        if let Some(block) = &mut self.state.resource_block
1796            && matches!(
1797                block.cause,
1798                crate::run::ResourceBlockCause::Cancelled { .. }
1799            )
1800        {
1801            block.cause = crate::run::ResourceBlockCause::Cancelled {
1802                park: cause.clone(),
1803            };
1804        }
1805        let why = match cause.as_ref().map(ParkCause::label) {
1806            Some(reason) => format!(
1807                "parked after `{}` ({reason}) — resume to carry on from here",
1808                self.state.status.as_str()
1809            ),
1810            None => format!(
1811                "parked after `{}` — resume to carry on from here",
1812                self.state.status.as_str()
1813            ),
1814        };
1815        self.state.event("park", why);
1816        self.state
1817            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1818        self.state.parked = true;
1819        self.state.save()?;
1820        Ok(true)
1821    }
1822
1823    /// Only a new explicit operator approval may retry failed preparation.
1824    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1825        if self.state.prep_failure.is_none() {
1826            return Ok(());
1827        }
1828        let failed_at = self
1829            .state
1830            .events
1831            .iter()
1832            .rev()
1833            .find(|event| event.node == "prep")
1834            .map_or(self.state.updated_at, |event| event.at);
1835        anyhow::ensure!(
1836            approval_at > failed_at,
1837            "preparation retry requires a new explicit release after the failure"
1838        );
1839        self.state.prep_failure = None;
1840        self.state.prep_complete = Some(false);
1841        self.state
1842            .event("prep-retry", "operator approved retry of preparation");
1843        Ok(())
1844    }
1845
1846    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1847    pub fn on_pause(&mut self, pause: Pause) {
1848        self.pause = pause;
1849    }
1850
1851    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1852    /// scheduler asking this one run - and no other - to park so a task
1853    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1854    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1855    /// handle.
1856    pub fn watch_interrupt(&mut self, pause: Pause) {
1857        self.interrupt = pause;
1858    }
1859
1860    /// Abandon this run's own open questions, once `status` has actually
1861    /// settled rather than merely paused.
1862    ///
1863    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1864    /// either back up with the candidates, the review round and the seat
1865    /// sessions already on disk, so a question an implementer asked mid-round
1866    /// may still get a real answer read by a real resume. Only the statuses
1867    /// `resumable` excludes are actually final: the run merged, it reached
1868    /// `Ready` with nothing left to do, it failed outright with no
1869    /// established point to continue from, or every candidate agreed, with
1870    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1871    /// every one of those the seat that asked is gone for good, exactly like
1872    /// the run being deleted under `magi run rm` - so the same cleanup
1873    /// applies, worded for what actually happened instead of "the run was
1874    /// deleted".
1875    ///
1876    /// Best-effort and silent on success: called from every place `status`
1877    /// can land on one of those three, including ones a resumed run revisits,
1878    /// so it must cost nothing when there was nothing open to begin with.
1879    fn settle_questions(&mut self) {
1880        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1881            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1882        }
1883    }
1884
1885    /// The tail of the graph after a trustworthy tally: fold losers, review,
1886    /// gate, merge, and persist.
1887    async fn finish_after_tally(&mut self) -> Result<()> {
1888        // Before the losers are folded: their branches are what the graft
1889        // reads from.
1890        self.graft().await?;
1891        if self.resource_halt()? {
1892            return Ok(());
1893        }
1894        self.fold_losers().await?;
1895        // Before review starts, and again right before the gate: a run's
1896        // review rounds can themselves take long enough for the base to move
1897        // a second time, and the gate is the one node whose "green" gets
1898        // acted on.
1899        self.sync_to_base().await?;
1900        if self.hold_base_sync().await? || self.resource_halt()? {
1901            return Ok(());
1902        }
1903        if self.state.status == RunStatus::AlreadyInBase {
1904            return Ok(());
1905        }
1906        self.review_loop().await?;
1907        if self.resource_halt()? {
1908            return Ok(());
1909        }
1910        self.sync_to_base().await?;
1911        if self.hold_base_sync().await? || self.resource_halt()? {
1912            return Ok(());
1913        }
1914        if self.state.status == RunStatus::AlreadyInBase {
1915            return Ok(());
1916        }
1917        self.gate().await?;
1918        self.merge().await?;
1919        if self.resource_halt()? {
1920            return Ok(());
1921        }
1922        self.state.save()?;
1923        Ok(())
1924    }
1925
1926    // ---------------------------------------------------------------- prep
1927
1928    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1929    /// run with the step and its output in the event; it is never swallowed.
1930    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1931    /// resume starts over instead of finding candidates and missing seats.
1932    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1933        let result = match ensure_setup_config(&mut self.state, repo).await {
1934            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1935            Err(e) => Err(e),
1936        };
1937        if let Err(e) = result {
1938            git::worktree_remove_clean(repo, wt).await.ok();
1939            if in_prep {
1940                self.rollback_prep(repo).await;
1941            }
1942            self.state.status = RunStatus::Blocked;
1943            self.state.event(
1944                "setup",
1945                format!("worktree setup failed in {}: {e:#}", wt.display()),
1946            );
1947            self.state.save()?;
1948            return Err(e);
1949        }
1950        Ok(())
1951    }
1952
1953    /// Undo everything `prep` created, so a resume re-runs it whole.
1954    async fn rollback_prep(&mut self, repo: &Path) {
1955        let root = self.state.worktree_root();
1956        for c in std::mem::take(&mut self.state.candidates) {
1957            if matches!(
1958                git::worktree_remove_clean(repo, &c.worktree).await,
1959                Ok(true)
1960            ) {
1961                git::branch_delete(repo, &c.branch).await.ok();
1962            } else {
1963                self.state.candidates.push(c);
1964            }
1965        }
1966        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1967        for j in 1..=seats {
1968            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1969                .await
1970                .ok();
1971        }
1972        for k in 1..=self.state.config.graph.advisors {
1973            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1974                .await
1975                .ok();
1976        }
1977    }
1978
1979    async fn prep(&mut self) -> Result<()> {
1980        if let Some(reason) = &self.state.prep_failure {
1981            bail!(
1982                "candidate preparation previously failed: {reason}; correct the source before retrying"
1983            );
1984        }
1985        if self
1986            .state
1987            .prep_complete
1988            .unwrap_or(!self.state.candidates.is_empty())
1989        {
1990            return Ok(());
1991        }
1992        let repo = self.state.repo.clone();
1993        // Includes rollback, config, refs, checkout creation and setup. No
1994        // repository mutation may precede the real shared lock acquisition.
1995        let _coordination = self.coordinate(&repo).await?;
1996        let base = self.state.base_commit.clone();
1997        let plan = self.state.sources.plan(&repo, &base).await?;
1998        self.rollback_prep(&repo).await;
1999        if !self.state.candidates.is_empty() {
2000            bail!(
2001                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2002            );
2003        }
2004        self.state.prep_complete = Some(false);
2005        self.state.status = RunStatus::Prep;
2006        self.state.save()?;
2007        let repo = self.state.repo.clone();
2008        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2009        let root = self.state.worktree_root();
2010        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2011
2012        // The hook is the write-time half of the blindness contract; the
2013        // presentation filter in `blind` is the half that cannot be bypassed.
2014        let hooks_dir = self.state.dir().join("hooks");
2015        if self.state.config.blind.commit_msg_hook {
2016            std::fs::create_dir_all(&hooks_dir)
2017                .with_context(|| format!("create {}", hooks_dir.display()))?;
2018            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2019            let path = hooks_dir.join("commit-msg");
2020            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2021            make_executable(&path)?;
2022            // Ref-counted rather than a plain idempotent set: with more than
2023            // one run able to be in flight in the same repository at once
2024            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2025            // true?" check cannot tell "another run of mine still needs
2026            // this" from "nobody does", and the run that happens to finish
2027            // first would disable the hook out from under a sibling still
2028            // relying on it.
2029            git::acquire_worktree_config(&repo).await?;
2030            self.state.enabled_worktree_config = true;
2031        }
2032
2033        // `[worktree] setup` hides its products per worktree, which needs the
2034        // same per-worktree config the hook does. Held until fold, like it.
2035        ensure_setup_config(&mut self.state, &repo).await?;
2036
2037        for (index, (spec, label)) in self
2038            .roles
2039            .implementers
2040            .clone()
2041            .into_iter()
2042            .zip(labels)
2043            .enumerate()
2044        {
2045            let branch = self.state.branch_for(label);
2046            let worktree = root.join(format!("cand-{label}"));
2047            // Persist intent before Git can leave a partially created tree.
2048            // Candidate registration follows setup, so it cannot inventory
2049            // crashes or refusals during creation itself.
2050            self.state.event(
2051                "prep-intent",
2052                serde_json::json!({
2053                    "branch": branch,
2054                    "worktree": worktree,
2055                    "start": start,
2056                    "picks": plan.picks,
2057                })
2058                .to_string(),
2059            );
2060            self.state.save()?;
2061            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2062            if self.state.config.blind.commit_msg_hook {
2063                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2064            }
2065            git::local_exclude(&worktree, "/.magi/").await?;
2066            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2067                // Nothing of this prep may survive: a resume must start over,
2068                // not find half the candidates and skip the rest.
2069                git::worktree_remove_clean(&repo, &worktree).await.ok();
2070                git::branch_delete(&repo, &branch).await.ok();
2071                self.rollback_prep(&repo).await;
2072                self.state.status = RunStatus::Blocked;
2073                self.state
2074                    .event("prep", format!("worktree setup failed: {e:#}"));
2075                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2076                self.state.save()?;
2077                return Err(e);
2078            }
2079            self.state.candidates.push(Candidate {
2080                index,
2081                label,
2082                agent: spec.id.clone(),
2083                branch,
2084                worktree,
2085                summary: String::new(),
2086                stat: String::new(),
2087                files: 0,
2088                commits: 0,
2089                empty: false,
2090                failed: None,
2091                verified_noop: None,
2092                duration_ms: 0,
2093                folded: false,
2094            });
2095            self.state.save()?;
2096            for pick in &plan.picks {
2097                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2098                {
2099                    self.state.status = RunStatus::Blocked;
2100                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2101                    self.state
2102                        .event("prep", format!("cannot apply referenced commit: {e}"));
2103                    self.state.save()?;
2104                    return Err(e);
2105                }
2106            }
2107        }
2108
2109        for j in 1..=self.roles.judges.len() {
2110            let wt = root.join(format!("judge-{j}"));
2111            if !wt.exists() {
2112                git::worktree_add_detached(&repo, &wt, &base).await?;
2113                self.setup_seat_worktree(&repo, &wt, true).await?;
2114            }
2115        }
2116
2117        // Disposable, detached checkouts for the design-deliberation stage's
2118        // advisor seats — the same shape as the judges' above, at the same
2119        // base commit, since advisors also only ever read. Sized off the
2120        // configured count directly rather than a resolved roster: unlike
2121        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2122        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2123        // `prep` has no `ResolvedRoles` field to read a count from here.
2124        if self.state.config.graph.advise {
2125            for k in 1..=self.state.config.graph.advisors {
2126                let wt = root.join(format!("advisor-{k}"));
2127                if !wt.exists() {
2128                    git::worktree_add_detached(&repo, &wt, &base).await?;
2129                    self.setup_seat_worktree(&repo, &wt, true).await?;
2130                }
2131            }
2132        }
2133
2134        // A judge cannot tell it is looking at its own patch — the seats keep
2135        // separate conversations — but a panel that shares agents with the
2136        // field is less independent than it looks, and that is worth saying out
2137        // loud once per run rather than leaving it in the config.
2138        let authors: Vec<&str> = self
2139            .roles
2140            .implementers
2141            .iter()
2142            .map(|a| a.id.as_str())
2143            .collect();
2144        let overlap: Vec<String> = self
2145            .roles
2146            .judges
2147            .iter()
2148            .enumerate()
2149            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2150            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2151            .collect();
2152        if !overlap.is_empty() {
2153            let note = format!(
2154                "{} also authored a candidate; blind, but the panel is less \
2155                 independent than {} distinct agents would be",
2156                overlap.join(", "),
2157                self.roles.judges.len()
2158            );
2159            self.state.event("prep", note);
2160        }
2161
2162        self.state.event(
2163            "prep",
2164            format!(
2165                "{} candidates, {} judges, base {} ({})",
2166                self.state.candidates.len(),
2167                self.roles.judges.len(),
2168                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2169                self.state.base_branch
2170            ),
2171        );
2172        self.state.prep_complete = Some(true);
2173        self.state.status = RunStatus::Implementing;
2174        self.state.save()?;
2175        Ok(())
2176    }
2177
2178    // -------------------------------------------------------------- advise
2179
2180    /// The design-deliberation stage: independent, read-only advisor seats
2181    /// each sketch a design before any implementer touches the repository,
2182    /// and (when at least one produced a usable proposal) a synthesis seat
2183    /// blends them into a brief `implement` carries in every candidate's
2184    /// prompt.
2185    ///
2186    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2187    /// advisors` is the proposal count. Everything here is best-effort and
2188    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2189    /// that cannot reach quota, or a synthesis seat that produced nothing
2190    /// usable all leave `implement` exactly as it was before this stage
2191    /// existed — the task instruction alone — rather than failing the whole
2192    /// competition over an enrichment stage. Every outcome is still recorded
2193    /// as an event, so a run that got nothing from this stage says why.
2194    ///
2195    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2196    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2197    /// resumed run whose stage failed would re-run it, and re-spend the
2198    /// agent calls, on every reentry before `implement`.
2199    ///
2200    /// Also skipped once any candidate shows implementation progress — the
2201    /// exact predicate `implement` itself uses to decide a candidate is no
2202    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2203    /// is not enough: a run created by an older binary that predates this
2204    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2205    /// an already-`Implementing`-or-later run under this build would
2206    /// otherwise walk straight back through `prep` (a no-op once candidates
2207    /// exist) into this node and spawn every advisor seat against worktrees
2208    /// `prep` never recreated — after implementation has already started,
2209    /// which is exactly the invariant this stage exists to guarantee.
2210    async fn advise(&mut self) -> Result<()> {
2211        let implement_untouched = self
2212            .state
2213            .candidates
2214            .iter()
2215            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2216        if !self.state.config.graph.advise || self.state.advise_attempted {
2217            return Ok(());
2218        }
2219        if !implement_untouched {
2220            self.state.event(
2221                "advise",
2222                "skipping the design-deliberation stage: at least one \
2223                 candidate already shows implementation progress, so this \
2224                 run is past the point the stage exists to run before"
2225                    .to_owned(),
2226            );
2227            self.state.advise_attempted = true;
2228            self.state.save()?;
2229            return Ok(());
2230        }
2231        let run_id = self.state.id.clone();
2232        let prompts = self.state.config.prompts.clone();
2233        let instruction = self.state.instruction.clone();
2234        let language = self.state.config.graph.language.clone();
2235        let root = self.state.worktree_root();
2236        let n = self.state.config.graph.advisors;
2237        let where_recorded = self.state.dir().join("run.json");
2238
2239        let seats = match self.state.config.advisors() {
2240            Ok(seats) if !seats.is_empty() => seats,
2241            Ok(_) => {
2242                self.state.event(
2243                    "advise",
2244                    format!(
2245                        "[graph] advisors is 0; skipping the design-deliberation \
2246                         stage and continuing without a synthesis brief (see {})",
2247                        where_recorded.display()
2248                    ),
2249                );
2250                self.state.advise_attempted = true;
2251                self.state.save()?;
2252                return Ok(());
2253            }
2254            Err(e) => {
2255                self.state.event(
2256                    "advise",
2257                    format!(
2258                        "could not resolve advisor seats ({e:#}); continuing \
2259                         without a design-deliberation brief (see {})",
2260                        where_recorded.display()
2261                    ),
2262                );
2263                self.state.advise_attempted = true;
2264                self.state.save()?;
2265                return Ok(());
2266            }
2267        };
2268
2269        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2270        let artifacts = agent::artifacts_dir(&self.state.dir());
2271        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2272
2273        let mut jobs = Vec::new();
2274        for (i, spec) in seats.iter().cloned().enumerate() {
2275            let seat_key = format!("advisor-{}", i + 1);
2276            let seat = self.seat(&seat_key, &spec.id);
2277            jobs.push(SeatJob {
2278                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2279                spec,
2280                seat,
2281                cwd: worktrees[i % worktrees.len()].clone(),
2282                timeout,
2283                allow_write: false,
2284                sessions: false,
2285                artifacts: artifacts.clone(),
2286                stem: seat_key,
2287                handover: None,
2288            });
2289        }
2290
2291        self.state.event(
2292            "advise",
2293            format!(
2294                "{} advisor seat(s) sketching a design in parallel",
2295                jobs.len()
2296            ),
2297        );
2298        let mut quota_losses = Vec::new();
2299        let cache = self.state.config.cache_dir();
2300        let ctx = WaveCtx {
2301            carry_seats: false,
2302            pauses: Some((&self.pause, &self.interrupt)),
2303            run: &run_id,
2304            node: "advise",
2305            prompts: &prompts,
2306            cache: cache.as_deref(),
2307            round: None,
2308        };
2309        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2310        let results = ask_json_wave::<Proposal>(
2311            jobs,
2312            Arc::clone(&self.sem),
2313            self.state.config.graph.retries,
2314            &advisor_roster,
2315            &ctx,
2316            &mut quota_losses,
2317            &mut self.state,
2318            &|p: &Proposal| p.validate(),
2319        )
2320        .await;
2321        if self.state.resource_block.is_some() {
2322            return self.stop_for_resource();
2323        }
2324        self.state.quota.extend(quota_losses);
2325
2326        let mut records = Vec::with_capacity(results.len());
2327        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2328            let agent_id = seat.agent.clone();
2329            self.state.seats.insert(seat.key.clone(), seat);
2330            match res {
2331                Ok((proposal, out)) => {
2332                    self.state
2333                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2334                    records.push(advise::AdvisorRecord::proposed(
2335                        i + 1,
2336                        agent_id,
2337                        proposal,
2338                        out.duration_ms,
2339                    ));
2340                }
2341                Err(e) => {
2342                    self.state.event(
2343                        "advise",
2344                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2345                    );
2346                    records.push(advise::AdvisorRecord::failed(
2347                        i + 1,
2348                        agent_id,
2349                        e.to_string(),
2350                    ));
2351                }
2352            }
2353        }
2354
2355        let mut advice = advise::Advice {
2356            records,
2357            synthesis: None,
2358        };
2359        if advice.proposals().is_empty() {
2360            self.state.event(
2361                "advise",
2362                "no advisor produced a usable proposal; continuing without a \
2363                 synthesis brief"
2364                    .to_owned(),
2365            );
2366        } else {
2367            match self
2368                .synthesize_brief(
2369                    &advice,
2370                    &instruction,
2371                    &language,
2372                    &worktrees[0],
2373                    &artifacts,
2374                    &run_id,
2375                    &prompts,
2376                    cache.as_deref(),
2377                )
2378                .await
2379            {
2380                Ok(Some(text)) => {
2381                    self.state.event(
2382                        "advise",
2383                        "synthesized a design brief for the implementer".to_owned(),
2384                    );
2385                    advice.synthesis = Some(text);
2386                }
2387                Ok(None) => {
2388                    self.state.event(
2389                        "advise",
2390                        "the synthesis seat produced nothing usable; continuing \
2391                         without a design brief"
2392                            .to_owned(),
2393                    );
2394                }
2395                Err(e) => {
2396                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2397                        return Err(e);
2398                    }
2399                    self.state.event(
2400                        "advise",
2401                        format!("could not synthesize a design brief: {e:#}"),
2402                    );
2403                }
2404            }
2405        }
2406        advise::apply_reflection(&mut advice);
2407
2408        self.state.advice = Some(advice);
2409        self.state.advise_attempted = true;
2410        self.state.save()?;
2411        Ok(())
2412    }
2413
2414    /// The synthesis seat: reads every advisor's proposal and blends them
2415    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2416    /// out of [`Runner::advise`] only for readability — it is not called
2417    /// anywhere else.
2418    ///
2419    /// Picked the same way [`crate::talk`]'s standing conversation and
2420    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2421    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2422    /// order (a claude seat, else the first runnable agent in roster order)
2423    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2424    /// in [`crate::config`] for why a dedicated field exists here at all.
2425    #[allow(clippy::too_many_arguments)]
2426    async fn synthesize_brief(
2427        &mut self,
2428        advice: &advise::Advice,
2429        instruction: &str,
2430        language: &str,
2431        cwd: &Path,
2432        artifacts: &Path,
2433        run_id: &str,
2434        prompts: &Prompts,
2435        cache: Option<&Path>,
2436    ) -> Result<Option<String>> {
2437        let chain = agent::pick_chain(
2438            &self.state.config.agents,
2439            self.state.config.roles.synthesizer.as_ref(),
2440            &agent::installed,
2441            "synthesizer",
2442        )?;
2443        let proposals = advice.proposals();
2444        let mut prompt = prompt::with_overlay(
2445            prompt::synthesize_brief(instruction, &proposals, language),
2446            prompts.overlay("advise"),
2447        );
2448        if cache.is_some() {
2449            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2450            // below — see `prompt::build_cache_note`'s doc for why telling a
2451            // read-only seat to build through the shared cache is exactly how
2452            // a sandbox's write refusal gets misread as a defect.
2453            prompt.push('\n');
2454            prompt.push_str(&prompt::build_cache_note("advise", false));
2455        }
2456        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2457        // Each id is tried once, in order; a quota hit, error or unusable
2458        // answer moves to the next. The seat is single-turn (`sessions:
2459        // false`) and the prompt is the whole context, so a fallback agent
2460        // needs nothing carried over.
2461        let mut last = None;
2462        for (n, spec) in chain.iter().enumerate() {
2463            if n > 0 {
2464                self.state
2465                    .event("advise", format!("synthesis falling back to {}", spec.id));
2466            }
2467            let mut seat = self.seat("advise-synthesis", &spec.id);
2468            let wait = self.repository_wait();
2469            let outcome = wait
2470                .scope(agent::invoke(
2471                    spec,
2472                    &mut seat,
2473                    &Invocation {
2474                        coordination: self.state.config.coordination.as_ref(),
2475                        fetched_base: None,
2476                        cwd,
2477                        prompt: &prompt,
2478                        timeout,
2479                        allow_write: false,
2480                        sessions: false,
2481                        artifacts,
2482                        stem: &if n == 0 {
2483                            "advise-synthesis".to_owned()
2484                        } else {
2485                            format!("advise-synthesis-{}", spec.id)
2486                        },
2487                        run: run_id,
2488                        node: "advise",
2489                        cache_dir: None,
2490                        attachments: &[],
2491                        writable: &[],
2492                    },
2493                ))
2494                .await;
2495            if outcome.is_ok() {
2496                self.state.seats.insert(seat.key.clone(), seat);
2497            }
2498            let advance = agent::chain_advances(&outcome);
2499            last = Some(outcome);
2500            if !advance {
2501                break;
2502            }
2503        }
2504        // Exhausted: the last attempt's result is what a single failed seat
2505        // would have produced.
2506        let out = last.expect("a chain holds at least one agent")?;
2507        if !out.usable() {
2508            return Ok(None);
2509        }
2510        let text =
2511            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2512        Ok((!text.trim().is_empty()).then_some(text))
2513    }
2514
2515    // ----------------------------------------------------------- implement
2516
2517    async fn implement(&mut self) -> Result<()> {
2518        self.persist_candidates().await?;
2519        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2520        // agent files with `magi task add` name the run that paid for it. The
2521        // prompt overlay is cloned alongside it because the waves borrow it
2522        // while `self` is mutably borrowed by the node's own bookkeeping.
2523        let run_id = self.state.id.clone();
2524        let prompts = self.state.config.prompts.clone();
2525        let todo: Vec<usize> = self
2526            .state
2527            .candidates
2528            .iter()
2529            .enumerate()
2530            .filter(|(_, c)| {
2531                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2532                    && c.failed.is_none()
2533                    && !c.empty
2534            })
2535            .map(|(i, _)| i)
2536            .collect();
2537        if todo.is_empty() {
2538            return self.after_implement();
2539        }
2540        self.state.status = RunStatus::Implementing;
2541
2542        let language = self.state.config.graph.language.clone();
2543        let github_language = self.state.config.graph.github_language.clone();
2544        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2545        let sessions = self.state.config.graph.sessions;
2546        let artifacts = agent::artifacts_dir(&self.state.dir());
2547        // The design-deliberation stage's blended brief, when `advise` found
2548        // one — carried into every implementer's prompt the same way
2549        // regardless of which candidate it is.
2550        let brief = self
2551            .state
2552            .advice
2553            .as_ref()
2554            .and_then(|a| a.synthesis.as_deref())
2555            .map(str::to_owned);
2556        let attachments = self.state.attachments.clone();
2557
2558        let mut jobs = Vec::new();
2559        for &i in &todo {
2560            let (index, label, worktree) = {
2561                let c = &self.state.candidates[i];
2562                (c.index, c.label, c.worktree.clone())
2563            };
2564            let spec = self.roles.implementers[index].clone();
2565            let seat_key = format!("impl-{label}");
2566            let seat = self.seat(&seat_key, &spec.id);
2567            let instruction = seeded_instruction(&self.state);
2568            jobs.push(SeatJob {
2569                spec,
2570                seat,
2571                prompt: prompt::implement_in(
2572                    &instruction,
2573                    &worktree.to_string_lossy(),
2574                    &language,
2575                    &github_language,
2576                    brief.as_deref(),
2577                    &attachments,
2578                ),
2579                cwd: worktree,
2580                timeout,
2581                allow_write: true,
2582                sessions,
2583                artifacts: artifacts.clone(),
2584                stem: format!("impl-{label}"),
2585                handover: None,
2586            });
2587        }
2588
2589        self.state.event(
2590            "implement",
2591            format!("{} candidates in parallel", jobs.len()),
2592        );
2593        // Kept so a seat whose CLI hung up can be asked again from the same
2594        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2595        // can update a seat's own entry once a fallback agent takes it over —
2596        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2597        // whichever agent actually answered, not the one that quota'd out.
2598        let mut sent = jobs.clone();
2599        let cache = self.state.config.cache_dir();
2600        let ctx = WaveCtx {
2601            carry_seats: false,
2602            pauses: Some((&self.pause, &self.interrupt)),
2603            run: &run_id,
2604            node: "implement",
2605            prompts: &prompts,
2606            cache: cache.as_deref(),
2607            round: None,
2608        };
2609        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2610        if !wave_blocked(&results) {
2611            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2612                .await;
2613            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2614                .await?;
2615            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2616                .await;
2617        }
2618        // Do not finalize failed candidates when the entire wave needs machine
2619        // recovery: leaving them pending resumes this node with its work intact.
2620        if !results.is_empty()
2621            && results
2622                .iter()
2623                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
2624        {
2625            let losses: Vec<_> = results
2626                .iter()
2627                .map(|(_, seat, out)| {
2628                    let AgentOutcome::Quota(o) = out else {
2629                        unreachable!()
2630                    };
2631                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
2632                })
2633                .collect();
2634            self.state.quota.extend(losses.clone());
2635            for (_, seat, _) in results {
2636                self.state.seats.insert(seat.key.clone(), seat);
2637            }
2638            for &i in &todo {
2639                let label = self.state.candidates[i].label;
2640                self.state.interrupted_implementations.insert(label);
2641                self.state
2642                    .candidate_preservation
2643                    .insert(label, "machine outage awaiting persistence".into());
2644            }
2645            // Preserve partial edits too, while keeping every interrupted seat
2646            // pending even when its tree already contains an inherited patch.
2647            block_machine_outage(&mut self.state, "implement", &losses);
2648            self.state.save()?;
2649            self.persist_candidates().await?;
2650            for &i in &todo {
2651                self.state
2652                    .interrupted_implementations
2653                    .insert(self.state.candidates[i].label);
2654            }
2655            return self.stop_for_resource();
2656        }
2657        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2658
2659        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2660            && results
2661                .iter()
2662                .any(|(_, _, out)| FailClass::of(out).is_some());
2663        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2664            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2665                // Executed turns interrupted by disk pressure already carry
2666                // a preservation barrier from wave. Admission-only waits do
2667                // not. Keep the interrupted seat's session for recovery too.
2668                if self
2669                    .state
2670                    .candidate_preservation
2671                    .contains_key(&self.state.candidates[i].label)
2672                {
2673                    self.state.seats.insert(seat.key.clone(), seat);
2674                }
2675                continue;
2676            }
2677            let seat_key = seat.key.clone();
2678            // A quota fallback (`resume_seat_handovers`) may have handed this
2679            // seat to a different agent than the one `prep` recorded on the
2680            // candidate; the stats tables and any later fixer-defaults-to-
2681            // winner's-author lookup must credit whoever actually answered —
2682            // unless every fallback also quota'd out, in which case nobody
2683            // actually answered and crediting the last agent tried would
2684            // erase every earlier agent's own quota loss from the stats
2685            // tables instead of just this one seat's.
2686            let agent = seat.agent.clone();
2687            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2688            self.state.seats.insert(seat.key.clone(), seat);
2689            let label = self.state.candidates[i].label;
2690            self.state.interrupted_implementations.remove(&label);
2691
2692            let (summary, duration, failed, verified_claim) = match out {
2693                AgentOutcome::Ok(o) => {
2694                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2695                    let failed = (!o.usable()).then(|| {
2696                        if o.timed_out {
2697                            "agent timed out".to_owned()
2698                        } else {
2699                            format!(
2700                                "agent exited with {}",
2701                                crate::notices::exit_status(o.exit_code)
2702                            )
2703                        }
2704                    });
2705                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2706                    (text, o.duration_ms, failed, verified_claim)
2707                }
2708                // Left un-resumed by `resume_undelivered` (a dirty tree
2709                // already rescues the work, or there was no session left to
2710                // resume into) — reported like the ordinary failure it is,
2711                // never as if `o.text` (the CLI's raw error JSON) were an
2712                // answer.
2713                AgentOutcome::Dropped(o) => {
2714                    let why = o
2715                        .dropped
2716                        .as_ref()
2717                        .map(|d| d.why.as_str())
2718                        .unwrap_or("the CLI ended the stream without delivering its answer");
2719                    (
2720                        String::new(),
2721                        o.duration_ms,
2722                        Some(format!("the CLI dropped the stream ({why})")),
2723                        None,
2724                    )
2725                }
2726                AgentOutcome::Quota(o) => {
2727                    self.state.quota.push(QuotaLoss::from_report(
2728                        seat_key,
2729                        "implement",
2730                        o.quota.as_ref(),
2731                    ));
2732                    (
2733                        String::new(),
2734                        o.duration_ms,
2735                        Some("rate limited (quota); produced no change".to_owned()),
2736                        None,
2737                    )
2738                }
2739                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2740                    (String::new(), 0, Some(e), None)
2741                }
2742            };
2743
2744            // Save every completed turn before any fallible Git operation. A failed
2745            // rescue must not erase the reply or prevent collection of other seats.
2746            let c = &mut self.state.candidates[i];
2747            if !exhausted_the_fallback_chain {
2748                c.agent = agent;
2749            }
2750            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2751            c.duration_ms = duration;
2752            c.failed = failed;
2753            c.verified_noop = verified_claim;
2754            self.state
2755                .candidate_preservation
2756                .insert(label, "awaiting persistence".to_owned());
2757        }
2758        self.state.save()?;
2759        self.persist_candidates().await?;
2760
2761        if resource_blocked {
2762            return self.stop_for_resource();
2763        }
2764        if selected_failed {
2765            self.state.status = RunStatus::Blocked;
2766            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2767            self.state.save()?;
2768            bail!(
2769                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2770            );
2771        }
2772        self.after_implement()
2773    }
2774
2775    /// Retry only persistence after a completed implementation turn.
2776    async fn persist_candidates(&mut self) -> Result<()> {
2777        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2778        for label in labels {
2779            let i = self
2780                .state
2781                .candidates
2782                .iter()
2783                .position(|c| c.label == label)
2784                .context("pending candidate is missing")?;
2785            let c = self.state.candidates[i].clone();
2786            let interrupted = self.state.interrupted_implementations.contains(&label);
2787            let result = async {
2788                let wait = crate::repo_lock::WaitPolicy::new(
2789                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2790                    || false,
2791                );
2792                let _coordination =
2793                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2794                // Best effort evidence requires no index writes. A successful snapshot
2795                // never authorizes removal of a tree whose rescue failed.
2796                if let Err(e) = git::snapshot_uncommitted(
2797                    &c.worktree,
2798                    &self
2799                        .state
2800                        .dir()
2801                        .join("artifacts")
2802                        .join(format!("candidate-{label}-recovery")),
2803                )
2804                .await
2805                {
2806                    self.state.event(
2807                        "preserve",
2808                        format!("candidate {label} snapshot incomplete: {e:#}"),
2809                    );
2810                }
2811                let rescued = git::rescue_commit(
2812                    &c.worktree,
2813                    &format!("magi: candidate {label} (uncommitted work)"),
2814                )
2815                .await?;
2816                self.state.note_withheld("implement", &rescued.withheld);
2817                let base = &self.state.base_commit;
2818                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2819                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2820                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2821                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2822                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2823                if c.empty && commits > 0 && !patch.trim().is_empty()
2824                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2825                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2826                        || self.state.tally.is_some())
2827                {
2828                    // Keep the old classification and pending marker so every
2829                    // retry stops too. Retain decisions as recovery evidence;
2830                    // only a supported recovery operation may replace them.
2831                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2832                }
2833                let c = &mut self.state.candidates[i];
2834                c.commits = commits;
2835                c.stat = stat;
2836                c.files = files;
2837                let no_changes = commits == 0 || patch.trim().is_empty();
2838                c.empty = no_changes && !interrupted;
2839                if interrupted && no_changes {
2840                    c.failed = None;
2841                    c.verified_noop = None;
2842                }
2843                if !c.empty {
2844                    c.failed = None;
2845                    c.verified_noop = None;
2846                }
2847                self.state.event(
2848                    "preserve",
2849                    format!("candidate {label}: {files} files, {commits} commits"),
2850                );
2851                Ok::<_, anyhow::Error>(())
2852            }
2853            .await;
2854            match result {
2855                Ok(()) => {
2856                    self.state.candidate_preservation.remove(&label);
2857                    if !interrupted || self.state.candidates[i].files > 0 {
2858                        self.state.interrupted_implementations.remove(&label);
2859                    }
2860                }
2861                Err(e) => {
2862                    let reason = format!("candidate {label} preservation failed: {e:#}");
2863                    self.state.event("preserve", &reason);
2864                    self.state.candidate_preservation.insert(label, reason);
2865                }
2866            }
2867            if self.state.candidate_preservation.is_empty()
2868                && self.state.status == RunStatus::Blocked
2869                && self.state.candidates.iter().all(|c| c.failed.is_none())
2870            {
2871                // Persist the recovered phase with the cleared pending marker;
2872                // a crash between saves must not turn this into a selected-seat
2873                // failure hold on the next resume.
2874                self.state.status = RunStatus::Implementing;
2875                self.state.event(
2876                    "preserve",
2877                    "candidate persistence recovered; implementation can continue",
2878                );
2879            }
2880            self.state.save()?;
2881        }
2882        if !self.state.candidate_preservation.is_empty() {
2883            self.state.status = RunStatus::Blocked;
2884            self.state.save()?;
2885            bail!(
2886                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2887            );
2888        }
2889        Ok(())
2890    }
2891
2892    /// Ask again, once, for work a CLI did and then failed to hand over.
2893    ///
2894    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2895    /// status with an empty response and a usage report showing output tokens,
2896    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2897    /// seven minutes and 14,267 output tokens that arrived as an empty
2898    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2899    ///
2900    /// Two conditions, and both matter:
2901    ///
2902    /// - **Only when the tree is untouched.** Often the agent has already
2903    ///   written its files and only the closing message was lost; the rescue
2904    ///   commit below picks that up and there is nothing to ask for. Re-asking
2905    ///   then would pay for a second implementation of work already on disk.
2906    /// - **Once.** A CLI that drops one stream can drop the next, and this
2907    ///   node is the most expensive in the graph.
2908    ///
2909    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2910    /// dropped reply still carried its `conversation_id`, so the seat is asked
2911    /// to finish what it was doing rather than sent the whole task again. It
2912    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2913    /// node's - for the same reason a re-ranked judge does: restating finished
2914    /// work is not the work.
2915    ///
2916    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2917    /// same way until it resets, while an abandoned conversation is still
2918    /// there to be picked up.
2919    async fn resume_undelivered(
2920        &mut self,
2921        results: &mut [(usize, SeatState, AgentOutcome)],
2922        sent: &[SeatJob],
2923        prompts: &Prompts,
2924        run_id: &str,
2925    ) {
2926        for (wi, seat, out) in results.iter_mut() {
2927            let Some(dropped) = (match &*out {
2928                AgentOutcome::Dropped(o) => o.dropped.clone(),
2929                _ => None,
2930            }) else {
2931                continue;
2932            };
2933            let Some(job) = sent.get(*wi) else { continue };
2934            // Already on disk? Then only the closing message was lost.
2935            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2936                self.state.event(
2937                    "implement",
2938                    format!(
2939                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2940                         work is in the tree",
2941                        seat.key, dropped.output_tokens, dropped.why
2942                    ),
2943                );
2944                continue;
2945            }
2946            // The re-ask only makes sense as a resume: `resume_after_drop`
2947            // says nothing about the task, trusting the seat to still hold it.
2948            // Without a session to resume — sessions disabled, or this CLI's
2949            // drop shape happened not to carry a session id — that prompt
2950            // would open a brand-new conversation with no context at all,
2951            // which is worse than leaving this as the ordinary failure it
2952            // already is.
2953            if !has_context(&job.spec, seat, job.sessions) {
2954                self.state.event(
2955                    "implement",
2956                    format!(
2957                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2958                         is no session left to resume",
2959                        seat.key, dropped.output_tokens, dropped.why
2960                    ),
2961                );
2962                continue;
2963            }
2964            self.state.event(
2965                "implement",
2966                format!(
2967                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2968                     conversation",
2969                    seat.key, dropped.output_tokens, dropped.why
2970                ),
2971            );
2972            let mut retry = job.clone();
2973            retry.seat = seat.clone();
2974            retry.prompt = prompt::resume_after_drop(&dropped.why);
2975            retry.timeout = retry_budget(job.timeout, true);
2976            retry.stem = format!("{}-resume", job.stem);
2977            let cache = self.state.config.cache_dir();
2978            let ctx = WaveCtx {
2979                carry_seats: false,
2980                pauses: Some((&self.pause, &self.interrupt)),
2981                run: run_id,
2982                node: "implement",
2983                prompts,
2984                cache: cache.as_deref(),
2985                round: None,
2986            };
2987            let (resumed_seat, resumed) =
2988                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2989            *seat = resumed_seat;
2990            *out = resumed;
2991        }
2992    }
2993
2994    /// Fall an implement seat through to the next untried agent in the
2995    /// implementer roster when it lost to quota — or, since the handover was
2996    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2997    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2998    /// quota path itself being unchanged — instead of leaving the
2999    /// seat's loss final the moment one agent's account runs dry.
3000    ///
3001    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3002    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3003    /// truncates to the single slot rotation picked, so a solo task whose one
3004    /// implementer hits quota mid-run used to have nothing else to try. This
3005    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3006    /// unrotated roster — which is the only place the *other* candidates in
3007    /// the machine's roster still exist once `implementers` has been cut down
3008    /// to size.
3009    ///
3010    /// Walks forward from just past the seat's own original position in the
3011    /// roster, never wrapping back to the front: a later candidate slot (say
3012    /// `beta`, the roster's second entry) must fall through to the *next*
3013    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3014    /// almost certainly a different candidate's own agent already — and once
3015    /// the roster's tail is exhausted there is nothing left to fall through
3016    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3017    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3018    /// let this retry that id forever. The loop keeps falling through until
3019    /// an attempt lands something other than `Quota` or the roster's tail
3020    /// runs out of untried ids, at which point the seat is left exactly as
3021    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3022    /// `QuotaLoss` recorded, the candidate failed/empty.
3023    ///
3024    /// `sent` is taken mutably and updated with the fallback agent's spec:
3025    /// `resume_unconfirmed_commands`, which runs after this and also reads
3026    /// `sent`, must see whichever agent actually ended up answering the seat
3027    /// — reading the stale, original spec there would check session
3028    /// eligibility against the wrong CLI and could hand a fallback agent's
3029    /// session id to the agent that just lost the seat to quota.
3030    ///
3031    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3032    /// — `self.seat` only reuses state when the agent id is unchanged, so
3033    /// handing it a different id already gets this for free. Reusing the old
3034    /// seat would resume a different CLI's session as if it were a
3035    /// continuation of this one.
3036    ///
3037    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3038    /// a quota loss cuts an agent off mid-turn, so anything already in the
3039    /// tree is unfinished work, not a completed candidate a re-ask would pay
3040    /// for twice. A dirty tree is rescued into a commit first (the same
3041    /// neutral-identity rescue `implement`'s own outcome loop gives every
3042    /// candidate) so the next agent starts clean.
3043    ///
3044    /// The new agent gets the implementer's full prompt and full
3045    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3046    /// it has no session and no context, and is implementing the task from
3047    /// nothing, unlike a resumed drop which is only restating work already
3048    /// done.
3049    ///
3050    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3051    /// `implement` event, never into `self.state.quota` — that is what
3052    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3053    /// go unspent, and a seat that ultimately recovered on its second or
3054    /// third agent is not the stalled panel that check exists to catch. Only
3055    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3056    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3057    /// outcome loop already has — this helper never pushes to it itself.
3058    async fn resume_seat_handovers(
3059        &mut self,
3060        results: &mut [(usize, SeatState, AgentOutcome)],
3061        sent: &mut [SeatJob],
3062        prompts: &Prompts,
3063        run_id: &str,
3064    ) -> Result<()> {
3065        let instruction = seeded_instruction(&self.state);
3066        let language = self.state.config.graph.language.clone();
3067        let github_language = self.state.config.graph.github_language.clone();
3068        let brief = self
3069            .state
3070            .advice
3071            .as_ref()
3072            .and_then(|a| a.synthesis.as_deref())
3073            .map(str::to_owned);
3074        let attachments = self.state.attachments.clone();
3075        for (wi, seat, out) in results.iter_mut() {
3076            // Who holds the other candidate seats of this wave right now
3077            // (earlier handovers already written back to `sent`).
3078            let others: BTreeSet<String> = sent
3079                .iter()
3080                .enumerate()
3081                .filter(|(j, _)| j != wi)
3082                .map(|(_, j)| j.spec.id.clone())
3083                .collect();
3084            let Some(job) = sent.get_mut(*wi) else {
3085                continue;
3086            };
3087            // Where the seat's own original agent sits in the roster — the
3088            // fallback walk starts just past here, never at the front, so a
3089            // later candidate slot's quota loss does not fall back onto an
3090            // earlier slot's own agent.
3091            let start = self
3092                .roles
3093                .implementer_roster
3094                .iter()
3095                .position(|s| s.id == job.spec.id)
3096                .unwrap_or(0);
3097            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3098            let mut fallback_attempt = 0usize;
3099            let mut prev: Option<FailClass> = None;
3100            while let Some(cur) = FailClass::of(&*out) {
3101                if !should_hand_over(prev.as_ref(), &cur) {
3102                    break;
3103                }
3104                let Some(next) =
3105                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3106                        .cloned()
3107                else {
3108                    break;
3109                };
3110                tried.insert(next.id.clone());
3111                fallback_attempt += 1;
3112
3113                let coordination = self.coordinate(&job.cwd).await?;
3114                if let Ok(r) = git::rescue_commit(
3115                    &job.cwd,
3116                    &format!(
3117                        "magi: candidate {} (uncommitted work before {} fallback)",
3118                        seat.key,
3119                        if cur == FailClass::Quota {
3120                            "quota"
3121                        } else {
3122                            "handover"
3123                        }
3124                    ),
3125                )
3126                .await
3127                {
3128                    self.state.note_withheld("implement", &r.withheld);
3129                }
3130                drop(coordination);
3131
3132                record_handover(
3133                    &mut self.state,
3134                    "implement",
3135                    &seat.key,
3136                    &seat.agent,
3137                    &next.id,
3138                    &cur,
3139                    &fail_reason(&*out),
3140                );
3141                prev = Some(cur.clone());
3142
3143                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3144                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3145                // Kept in sync on `sent` itself, not just the local retry: a
3146                // later helper (`resume_unconfirmed_commands`) reads `sent`
3147                // after this one returns and must see whichever agent is now
3148                // occupying the seat, not the one that just quota'd out —
3149                // otherwise it would judge session/continuation eligibility
3150                // by the wrong CLI and could resend a fallback's session id
3151                // to the agent that lost it the seat in the first place.
3152                job.spec = next.clone();
3153                let mut retry = job.clone();
3154                retry.seat = new_seat;
3155                retry.prompt = prompt::implement_in(
3156                    &instruction,
3157                    &job.cwd.to_string_lossy(),
3158                    &language,
3159                    &github_language,
3160                    brief.as_deref(),
3161                    &attachments,
3162                );
3163                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3164                let cache = self.state.config.cache_dir();
3165                let ctx = WaveCtx {
3166                    carry_seats: false,
3167                    pauses: Some((&self.pause, &self.interrupt)),
3168                    run: run_id,
3169                    node: "implement",
3170                    prompts,
3171                    cache: cache.as_deref(),
3172                    round: None,
3173                };
3174                let (fallback_seat, fallback_out) = run_one(
3175                    retry,
3176                    Arc::clone(&self.sem),
3177                    &ctx,
3178                    &mut self.state,
3179                    fallback_attempt,
3180                )
3181                .await;
3182                *seat = fallback_seat;
3183                *out = fallback_out;
3184            }
3185        }
3186        Ok(())
3187    }
3188
3189    /// Ask an implement seat's own CLI to confirm what it started, once, when
3190    /// its reply reported a command whose completion status it never
3191    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3192    /// that does and does not mean.
3193    ///
3194    /// The completion contract this task asks for, extended to `implement`
3195    /// with the same signal `continue_fix_report` reads for the fixer,
3196    /// rather than a keyword search over the reply or a hard requirement on
3197    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3198    /// a candidate's CLI turn ended cleanly while a test run it had started
3199    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3200    /// named in it at all is untouched by this: `commands` is empty, so
3201    /// there is nothing to be unconfirmed.
3202    ///
3203    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3204    /// this is not about recovering edits that might already be on disk, it
3205    /// is about a result the seat itself never vouched for, which resuming
3206    /// asks for regardless of what the tree already holds. Bounded to one
3207    /// attempt for the same reason `resume_undelivered` is — this is the
3208    /// most expensive node in the graph — and a seat that still cannot
3209    /// confirm on that attempt is left as whatever its (possibly still
3210    /// unconfirmed) reply says; this does not invent a new "failed" reason
3211    /// for a candidate that otherwise produced a real, committed change.
3212    async fn resume_unconfirmed_commands(
3213        &mut self,
3214        results: &mut [(usize, SeatState, AgentOutcome)],
3215        sent: &[SeatJob],
3216        prompts: &Prompts,
3217        run_id: &str,
3218    ) {
3219        for (wi, seat, out) in results.iter_mut() {
3220            let AgentOutcome::Ok(o) = &*out else {
3221                continue;
3222            };
3223            if !has_unconfirmed_command(&o.commands) {
3224                continue;
3225            }
3226            let Some(job) = sent.get(*wi) else { continue };
3227            if !has_context(&job.spec, seat, job.sessions) {
3228                self.state.event(
3229                    "implement",
3230                    format!(
3231                        "{}: the reply named a command whose own CLI never confirmed the exit \
3232                         status of, but there is no session left to resume",
3233                        seat.key
3234                    ),
3235                );
3236                continue;
3237            }
3238            self.state.event(
3239                "implement",
3240                format!(
3241                    "{}: the reply named a command whose own CLI never confirmed the exit \
3242                     status of; resuming the conversation",
3243                    seat.key
3244                ),
3245            );
3246            let mut retry = job.clone();
3247            retry.seat = seat.clone();
3248            retry.prompt = prompt::resume_incomplete(
3249                "a command in your last reply had no confirmed exit status",
3250            );
3251            retry.timeout = retry_budget(job.timeout, true);
3252            retry.stem = format!("{}-confirm", job.stem);
3253            let cache = self.state.config.cache_dir();
3254            let ctx = WaveCtx {
3255                carry_seats: false,
3256                pauses: Some((&self.pause, &self.interrupt)),
3257                run: run_id,
3258                node: "implement",
3259                prompts,
3260                cache: cache.as_deref(),
3261                round: None,
3262            };
3263            let (resumed_seat, resumed) =
3264                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3265            *seat = resumed_seat;
3266            *out = resumed;
3267        }
3268    }
3269
3270    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3271    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3272    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3273    /// that motivated this.
3274    ///
3275    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3276    /// own nudge loop already covers for judge/review/vote seats, and not a
3277    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3278    /// implement seats: here the CLI turn genuinely finished while the node's
3279    /// own work — the fixer's account of what it did — had not. Gated purely
3280    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3281    /// reply, never on any wording in it, so a fixer whose valid, first-try
3282    /// `FixReport` happens to mention having waited on a background test is
3283    /// never resumed — the `Ok(report)` branch at the call site returns
3284    /// before this is ever invoked.
3285    ///
3286    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3287    /// attempt ([`retry_budget`]), nothing attempted once the session is
3288    /// gone, and a quota hit ends the loop immediately rather than retrying a
3289    /// rate limit that fails the same way again.
3290    async fn continue_fix_report(
3291        &mut self,
3292        mut seat: SeatState,
3293        parse_err: String,
3294        job: &SeatJob,
3295        prompts: &Prompts,
3296        run_id: &str,
3297        round: usize,
3298    ) -> (
3299        SeatState,
3300        Option<FixReport>,
3301        Option<String>,
3302        ContinuationRecord,
3303    ) {
3304        let mut last_err = parse_err;
3305        let mut cumulative_wait_ms = 0u64;
3306        let mut attempts = 0usize;
3307        loop {
3308            if !has_context(&job.spec, &seat, job.sessions) {
3309                self.state.event(
3310                    "fix",
3311                    format!(
3312                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3313                         session left to resume into"
3314                    ),
3315                );
3316                let outcome = if attempts == 0 {
3317                    ContinuationOutcome::NoSession
3318                } else {
3319                    ContinuationOutcome::Exhausted
3320                };
3321                return (
3322                    seat,
3323                    None,
3324                    Some(format!("unparsable fix report: {last_err}")),
3325                    ContinuationRecord {
3326                        attempts,
3327                        cumulative_wait_ms,
3328                        outcome,
3329                    },
3330                );
3331            }
3332            if attempts >= MAX_FIX_CONTINUATIONS {
3333                self.state.event(
3334                    "fix",
3335                    format!(
3336                        "round {round}: fixer's reply still had no adoption report after \
3337                         {attempts} continuation(s) ({last_err}); giving up"
3338                    ),
3339                );
3340                return (
3341                    seat,
3342                    None,
3343                    Some(format!(
3344                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3345                    )),
3346                    ContinuationRecord {
3347                        attempts,
3348                        cumulative_wait_ms,
3349                        outcome: ContinuationOutcome::Exhausted,
3350                    },
3351                );
3352            }
3353            attempts += 1;
3354            self.state.event(
3355                "fix",
3356                format!(
3357                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3358                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3359                ),
3360            );
3361            let mut retry = job.clone();
3362            retry.seat = seat.clone();
3363            retry.prompt = prompt::resume_incomplete(&last_err);
3364            retry.timeout = retry_budget(job.timeout, true);
3365            retry.stem = format!("{}-continue{attempts}", job.stem);
3366            let cache = self.state.config.cache_dir();
3367            let ctx = WaveCtx {
3368                carry_seats: false,
3369                pauses: Some((&self.pause, &self.interrupt)),
3370                run: run_id,
3371                node: "fix",
3372                prompts,
3373                cache: cache.as_deref(),
3374                round: Some(round),
3375            };
3376            let (resumed_seat, resumed_out) = run_one(
3377                retry,
3378                Arc::clone(&self.sem),
3379                &ctx,
3380                &mut self.state,
3381                attempts,
3382            )
3383            .await;
3384            seat = resumed_seat;
3385            match resumed_out {
3386                AgentOutcome::Ok(o) => {
3387                    cumulative_wait_ms += o.duration_ms;
3388                    match verdict::extract_json::<FixReport>(&o.text) {
3389                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3390                            self.state.event(
3391                                "fix",
3392                                format!(
3393                                    "round {round}: fixer's adoption report recovered after \
3394                                     {attempts} continuation(s)"
3395                                ),
3396                            );
3397                            return (
3398                                seat,
3399                                Some(report),
3400                                None,
3401                                ContinuationRecord {
3402                                    attempts,
3403                                    cumulative_wait_ms,
3404                                    outcome: ContinuationOutcome::Resumed,
3405                                },
3406                            );
3407                        }
3408                        // The report parsed, but this same reply's own
3409                        // CommandEvidence — the identical record `state.jobs`
3410                        // renders — names a command whose CLI never
3411                        // confirmed an exit status. Read together, that is
3412                        // not a resolved answer: keep nudging rather than
3413                        // accept a report standing next to a command the
3414                        // seat's own CLI cannot vouch for.
3415                        Ok(_) => {
3416                            last_err = "the reply parsed, but it reported a command whose own CLI \
3417                                 never confirmed an exit status"
3418                                .to_owned();
3419                        }
3420                        Err(e) => last_err = e.to_string(),
3421                    }
3422                }
3423                AgentOutcome::Quota(o) => {
3424                    cumulative_wait_ms += o.duration_ms;
3425                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3426                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3427                    self.state.quota.push(loss);
3428                    self.state.event(
3429                        "fix",
3430                        format!(
3431                            "round {round}: continuation rate limited (quota); not retrying now"
3432                        ),
3433                    );
3434                    return (
3435                        seat,
3436                        None,
3437                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3438                        ContinuationRecord {
3439                            attempts,
3440                            cumulative_wait_ms,
3441                            outcome: ContinuationOutcome::QuotaLost,
3442                        },
3443                    );
3444                }
3445                AgentOutcome::Dropped(o) => {
3446                    cumulative_wait_ms += o.duration_ms;
3447                    let why = o
3448                        .dropped
3449                        .as_ref()
3450                        .map(|d| d.why.as_str())
3451                        .unwrap_or("the CLI ended the stream without delivering its answer");
3452                    last_err = format!("the CLI dropped the stream ({why})");
3453                }
3454                AgentOutcome::Failed(e) => last_err = e,
3455                // The continuation was never started; the report stays lost
3456                // and the caller records that, without any handover.
3457                AgentOutcome::ResourceBlocked(e) => {
3458                    return (
3459                        seat,
3460                        None,
3461                        Some(format!("the continuation was not started: {e}")),
3462                        ContinuationRecord {
3463                            attempts: attempts.saturating_sub(1),
3464                            cumulative_wait_ms,
3465                            outcome: ContinuationOutcome::NoSession,
3466                        },
3467                    );
3468                }
3469            }
3470        }
3471    }
3472
3473    fn after_implement(&mut self) -> Result<()> {
3474        if !self.state.candidate_preservation.is_empty() {
3475            bail!("candidate preservation incomplete; judging is unsafe");
3476        }
3477        // Scan every candidate patch once the set is complete.
3478        if self.state.leaks.is_empty() {
3479            let cfg = self.state.config.blind.clone();
3480            let mut leaks = Vec::new();
3481            for c in &self.state.candidates {
3482                let Some(patch) =
3483                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3484                else {
3485                    continue;
3486                };
3487                leaks.extend(blind::scan(
3488                    &format!("candidate {} patch", c.label),
3489                    &patch,
3490                    &cfg.vendor_tokens,
3491                ));
3492            }
3493            if !leaks.is_empty() {
3494                let summary = leaks
3495                    .iter()
3496                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3497                    .collect::<Vec<_>>()
3498                    .join(", ");
3499                match cfg.on_leak {
3500                    LeakPolicy::Fail => {
3501                        self.state.status = RunStatus::Failed;
3502                        self.state
3503                            .event("blind", format!("vendor text in a patch: {summary}"));
3504                        self.state.leaks = leaks;
3505                        self.state.save()?;
3506                        self.settle_questions();
3507                        bail!(
3508                            "blind.on_leak = \"fail\" and vendor text reached a \
3509                             judged patch: {summary}"
3510                        );
3511                    }
3512                    LeakPolicy::Redact => self.state.event(
3513                        "blind",
3514                        format!("redacting vendor text for judging: {summary}"),
3515                    ),
3516                    LeakPolicy::Warn => self.state.event(
3517                        "blind",
3518                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3519                    ),
3520                }
3521                self.state.leaks = leaks;
3522            }
3523        }
3524
3525        if self.state.viable().is_empty() {
3526            if self.state.all_candidates_verified_noop() {
3527                // Every candidate agreed, with evidence the adoption guard
3528                // accepted, that nothing belongs in this worktree. That is
3529                // not the same fact as a candidate that simply failed to
3530                // write anything, and settling it as an ordinary `Failed`
3531                // (see `SCHEMA`'s doc for schema 10) is what let two of
3532                // task 391f's attempts burn a retry each re-discovering the
3533                // same already-landed fix. Terminal either way, so `judge`
3534                // must never run over an empty candidate set — unlike the
3535                // `Failed` branch below this returns `Ok`, not an error:
3536                // nothing here failed.
3537                self.state.status = RunStatus::VerifiedNoop;
3538                self.state.save()?;
3539                self.settle_questions();
3540                return Ok(());
3541            }
3542            self.state.status = RunStatus::Failed;
3543            self.state.save()?;
3544            self.settle_questions();
3545            bail!("no candidate produced a change; nothing to judge");
3546        }
3547        self.state.status = RunStatus::Judging;
3548        self.state.save()?;
3549        Ok(())
3550    }
3551
3552    // --------------------------------------------------------------- judge
3553
3554    async fn judge(&mut self) -> Result<()> {
3555        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3556        // agent files with `magi task add` name the run that paid for it. The
3557        // prompt overlay is cloned alongside it because the waves borrow it
3558        // while `self` is mutably borrowed by the node's own bookkeeping.
3559        let run_id = self.state.id.clone();
3560        let prompts = self.state.config.prompts.clone();
3561        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3562            return Ok(());
3563        }
3564        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3565        if viable.len() == 1 {
3566            // Recorded so this is a one-time event: `judgements` stays empty
3567            // either way, which without this flag is indistinguishable from
3568            // "not yet judged" on the next reentry — and status is left
3569            // untouched, so a later node's conclusion (e.g. `Blocked` after
3570            // the review budget ran out) survives a resume instead of being
3571            // clobbered back to `Judging` by this node running again.
3572            self.state.judge_skipped = true;
3573            self.state.event(
3574                "judge",
3575                format!(
3576                    "only candidate {} produced a change; judging skipped",
3577                    viable[0].label
3578                ),
3579            );
3580            self.state.save()?;
3581            return Ok(());
3582        }
3583        self.state.status = RunStatus::Judging;
3584
3585        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3586        let language = self.state.config.graph.language.clone();
3587        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3588        let sessions = self.state.config.graph.sessions;
3589        let artifacts = agent::artifacts_dir(&self.state.dir());
3590        let root = self.state.worktree_root();
3591        let base_short = short(&self.state.base_commit);
3592
3593        let mut jobs = Vec::new();
3594        let mut orders = Vec::new();
3595        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3596            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3597            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3598            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3599            let seat_key = format!("judge-{}", j + 1);
3600            let seat = self.seat(&seat_key, &spec.id);
3601            jobs.push(SeatJob {
3602                prompt: prompt::judge(
3603                    &self.state.instruction,
3604                    &views,
3605                    self.roles.judges.len(),
3606                    &base_short,
3607                    &language,
3608                ),
3609                spec,
3610                seat,
3611                cwd: root.join(format!("judge-{}", j + 1)),
3612                timeout,
3613                allow_write: false,
3614                sessions,
3615                artifacts: artifacts.clone(),
3616                stem: format!("judge-{}", j + 1),
3617                handover: None,
3618            });
3619        }
3620
3621        self.state.event(
3622            "judge",
3623            format!(
3624                "{} judges ranking {} candidates blind",
3625                jobs.len(),
3626                viable.len()
3627            ),
3628        );
3629        let labels_for_check = labels.clone();
3630        let mut quota_losses = Vec::new();
3631        let cache = self.state.config.cache_dir();
3632        let ctx = WaveCtx {
3633            carry_seats: false,
3634            pauses: Some((&self.pause, &self.interrupt)),
3635            run: &run_id,
3636            node: "judge",
3637            prompts: &prompts,
3638            cache: cache.as_deref(),
3639            round: None,
3640        };
3641        let results = ask_json_wave::<Ranking>(
3642            jobs,
3643            Arc::clone(&self.sem),
3644            self.state.config.graph.retries,
3645            &self.roles.judge_roster,
3646            &ctx,
3647            &mut quota_losses,
3648            &mut self.state,
3649            &move |r: &Ranking| r.validate(&labels_for_check),
3650        )
3651        .await;
3652        if self.state.resource_block.is_some() {
3653            return self.stop_for_resource();
3654        }
3655        self.state.quota.extend(quota_losses);
3656
3657        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3658            let agent_id = seat.agent.clone();
3659            self.state.seats.insert(seat.key.clone(), seat);
3660            let mut record = Judgement {
3661                judge: j + 1,
3662                seat: format!("judge-{}", j + 1),
3663                agent: agent_id,
3664                ranking: Vec::new(),
3665                reasons: BTreeMap::new(),
3666                confidence: None,
3667                strengths: BTreeMap::new(),
3668                order: orders[j].clone(),
3669                failed: None,
3670                duration_ms: 0,
3671            };
3672            match res {
3673                Ok((ranking, out)) => {
3674                    record.ranking = ranking.normalized();
3675                    record.reasons = ranking.reasons;
3676                    record.strengths = ranking.strengths;
3677                    record.confidence = ranking.confidence;
3678                    record.duration_ms = out.duration_ms;
3679                    self.state.event(
3680                        "judge",
3681                        format!(
3682                            "judge {} ranked {}",
3683                            j + 1,
3684                            record.ranking.iter().collect::<String>()
3685                        ),
3686                    );
3687                }
3688                Err(e) => {
3689                    record.failed = Some(e.to_string());
3690                    self.state
3691                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3692                }
3693            }
3694            self.state.judgements.push(record);
3695            self.state.save()?;
3696        }
3697        Ok(())
3698    }
3699
3700    // ---------------------------------------------------------- deliberate
3701
3702    async fn deliberate(&mut self) -> Result<()> {
3703        if self.hold_base_sync().await? {
3704            return Ok(());
3705        }
3706        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3707        // agent files with `magi task add` name the run that paid for it. The
3708        // prompt overlay is cloned alongside it because the waves borrow it
3709        // while `self` is mutably borrowed by the node's own bookkeeping.
3710        let run_id = self.state.id.clone();
3711        let prompts = self.state.config.prompts.clone();
3712        let rounds = self.state.config.graph.deliberate_rounds;
3713        // Resume unfinished rounds after an outage, but never reopen a tally.
3714        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
3715            return Ok(());
3716        }
3717        let tops: Vec<char> = self
3718            .state
3719            .judgements
3720            .iter()
3721            .filter_map(|j| j.ranking.first().copied())
3722            .collect();
3723        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3724            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3725                self.state.event(
3726                    "deliberate",
3727                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3728                );
3729            }
3730            self.state.status = RunStatus::Voting;
3731            self.state.save()?;
3732            return Ok(());
3733        }
3734
3735        self.state.status = RunStatus::Deliberating;
3736        self.state.event(
3737            "deliberate",
3738            format!(
3739                "split: first choices were {} — opening {rounds} round(s)",
3740                tops.iter().collect::<String>()
3741            ),
3742        );
3743
3744        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3745        let language = self.state.config.graph.language.clone();
3746        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3747        let sessions = self.state.config.graph.sessions;
3748        let artifacts = agent::artifacts_dir(&self.state.dir());
3749        let root = self.state.worktree_root();
3750        let base_short = short(&self.state.base_commit);
3751
3752        // Judges argue in sequence so that a turn can answer the one before it;
3753        // that is the difference between deliberation and three parallel
3754        // monologues.
3755        for round in self.state.deliberation.len() + 1..=rounds {
3756            let mut turns: Vec<DeliberationTurn> = Vec::new();
3757            let mut round_losses = Vec::new();
3758            let mut asked = 0;
3759            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3760                if self.state.judgements[j].failed.is_some() {
3761                    continue;
3762                }
3763                let seat_key = format!("judge-{}", j + 1);
3764                let spec = self.occupant(&seat_key, spec);
3765                let mut seat = self.seat(&seat_key, &spec.id);
3766                let transcript = self.transcript(&turns, j);
3767                let build = |context: Option<&str>| {
3768                    prompt::deliberate(
3769                        &self.state.instruction,
3770                        context,
3771                        &transcript,
3772                        round,
3773                        rounds,
3774                        &language,
3775                    )
3776                };
3777                let block = self.candidate_block(&viable, &base_short);
3778                let full = build(Some(&block));
3779                let text = if has_context(&spec, &seat, sessions) {
3780                    build(None)
3781                } else {
3782                    full.clone()
3783                };
3784                let job = SeatJob {
3785                    spec,
3786                    seat: seat.clone(),
3787                    prompt: text,
3788                    cwd: root.join(format!("judge-{}", j + 1)),
3789                    timeout,
3790                    allow_write: false,
3791                    sessions,
3792                    artifacts: artifacts.clone(),
3793                    stem: format!("delib-{round}-judge-{}", j + 1),
3794                    handover: Some(full),
3795                };
3796                let cache = self.state.config.cache_dir();
3797                let ctx = WaveCtx {
3798                    carry_seats: false,
3799                    pauses: Some((&self.pause, &self.interrupt)),
3800                    run: &run_id,
3801                    node: "deliberate",
3802                    prompts: &prompts,
3803                    cache: cache.as_deref(),
3804                    round: None,
3805                };
3806                // A turn is never nudged (`retries` 0): a failed seat is
3807                // handed to the next roster agent, which gets the full
3808                // context. An empty answer is a turn, not a failure.
3809                let mut losses = Vec::new();
3810                asked += 1;
3811                let mut results = ask_wave_with::<String>(
3812                    vec![job],
3813                    Arc::clone(&self.sem),
3814                    0,
3815                    &self.roles.judge_roster,
3816                    &ctx,
3817                    &mut losses,
3818                    &mut self.state,
3819                    &|text: &str| {
3820                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3821                    },
3822                )
3823                .await;
3824                if self.state.resource_block.is_some() {
3825                    return self.stop_for_resource();
3826                }
3827                round_losses.extend(losses.iter().cloned());
3828                self.state.quota.extend(losses);
3829                let (updated, res, _) = results.pop().expect("one job in, one result out");
3830                seat = updated;
3831                let agent_id = seat.agent.clone();
3832                self.state.seats.insert(seat.key.clone(), seat);
3833                let body = match res {
3834                    Ok((body, _)) => body,
3835                    // Skip the seat; a CLI's raw error JSON is never read as
3836                    // this judge's position.
3837                    Err(e) => {
3838                        self.state
3839                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3840                        continue;
3841                    }
3842                };
3843                let tentative = verdict::extract_json::<Position>(&body)
3844                    .ok()
3845                    .and_then(|p| p.tentative)
3846                    .and_then(|s| s.trim().chars().next())
3847                    .map(|c| c.to_ascii_uppercase());
3848                self.state.event(
3849                    "deliberate",
3850                    format!(
3851                        "round {round}: judge {} now favours {}",
3852                        j + 1,
3853                        tentative.map_or("—".to_owned(), |c| c.to_string())
3854                    ),
3855                );
3856                turns.push(DeliberationTurn {
3857                    judge: j + 1,
3858                    agent: agent_id,
3859                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3860                    tentative,
3861                });
3862            }
3863            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
3864                block_machine_outage(&mut self.state, "deliberate", &round_losses);
3865                return self.stop_for_resource();
3866            }
3867            self.state
3868                .deliberation
3869                .push(DeliberationRound { round, turns });
3870            self.state.save()?;
3871        }
3872
3873        self.state.status = RunStatus::Voting;
3874        self.state.save()?;
3875        Ok(())
3876    }
3877
3878    // ---------------------------------------------------------------- vote
3879
3880    async fn vote(&mut self) -> Result<()> {
3881        if self.hold_base_sync().await? {
3882            return Ok(());
3883        }
3884        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3885        // agent files with `magi task add` name the run that paid for it. The
3886        // prompt overlay is cloned alongside it because the waves borrow it
3887        // while `self` is mutably borrowed by the node's own bookkeeping.
3888        let run_id = self.state.id.clone();
3889        let prompts = self.state.config.prompts.clone();
3890        if !self.state.votes.is_empty() || self.state.tally.is_some() {
3891            return Ok(());
3892        }
3893        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3894        if viable.len() == 1 {
3895            return Ok(());
3896        }
3897        self.state.status = RunStatus::Voting;
3898
3899        let language = self.state.config.graph.language.clone();
3900        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3901        let sessions = self.state.config.graph.sessions;
3902        let artifacts = agent::artifacts_dir(&self.state.dir());
3903        let root = self.state.worktree_root();
3904        let base_short = short(&self.state.base_commit);
3905        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3906
3907        let mut jobs = Vec::new();
3908        let mut seats_at = Vec::new();
3909        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3910            if self
3911                .state
3912                .judgements
3913                .get(j)
3914                .is_some_and(|r| r.failed.is_some())
3915            {
3916                continue;
3917            }
3918            let seat_key = format!("judge-{}", j + 1);
3919            let spec = self.occupant(&seat_key, spec);
3920            let seat = self.seat(&seat_key, &spec.id);
3921            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3922            let text = if has_context(&spec, &seat, sessions) {
3923                prompt::final_vote(&viable, &language)
3924            } else {
3925                full.clone()
3926            };
3927            jobs.push(SeatJob {
3928                spec,
3929                seat,
3930                prompt: text,
3931                cwd: root.join(format!("judge-{}", j + 1)),
3932                timeout,
3933                allow_write: false,
3934                sessions,
3935                artifacts: artifacts.clone(),
3936                stem: format!("vote-judge-{}", j + 1),
3937                handover: Some(full),
3938            });
3939            seats_at.push(j);
3940        }
3941
3942        self.state.event(
3943            "vote",
3944            format!(
3945                "collecting {} final votes one by one, privately",
3946                jobs.len()
3947            ),
3948        );
3949        let allowed = viable.clone();
3950        let mut quota_losses = Vec::new();
3951        let cache = self.state.config.cache_dir();
3952        let ctx = WaveCtx {
3953            carry_seats: false,
3954            pauses: Some((&self.pause, &self.interrupt)),
3955            run: &run_id,
3956            node: "vote",
3957            prompts: &prompts,
3958            cache: cache.as_deref(),
3959            round: None,
3960        };
3961        let results = ask_json_wave::<FinalVote>(
3962            jobs,
3963            Arc::clone(&self.sem),
3964            self.state.config.graph.retries,
3965            &self.roles.judge_roster,
3966            &ctx,
3967            &mut quota_losses,
3968            &mut self.state,
3969            &move |v: &FinalVote| match v.label() {
3970                Some(c) if allowed.contains(&c) => Ok(()),
3971                other => bail!("vote {other:?} is not one of {allowed:?}"),
3972            },
3973        )
3974        .await;
3975        if self.state.resource_block.is_some() {
3976            return self.stop_for_resource();
3977        }
3978        self.state.quota.extend(quota_losses);
3979
3980        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3981            let agent_id = seat.agent.clone();
3982            self.state.seats.insert(seat.key.clone(), seat);
3983            let initial = self
3984                .state
3985                .judgements
3986                .get(j)
3987                .and_then(|r| r.ranking.first().copied());
3988            let mut record = VoteRecord {
3989                judge: j + 1,
3990                agent: agent_id,
3991                vote: None,
3992                reason: String::new(),
3993                changed: false,
3994            };
3995            match res {
3996                Ok((v, _)) => {
3997                    record.vote = v.label();
3998                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3999                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4000                    self.state.event(
4001                        "vote",
4002                        format!(
4003                            "judge {} voted {}{}",
4004                            j + 1,
4005                            record.vote.unwrap_or('?'),
4006                            if record.changed { " (changed)" } else { "" }
4007                        ),
4008                    );
4009                }
4010                Err(e) => {
4011                    self.state
4012                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4013                }
4014            }
4015            self.state.votes.push(record);
4016            self.state.save()?;
4017        }
4018        Ok(())
4019    }
4020
4021    // --------------------------------------------------------------- tally
4022
4023    fn tally(&mut self) -> Result<()> {
4024        if self.state.tally.is_some() {
4025            return Ok(());
4026        }
4027        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4028        let tops: Vec<char> = self
4029            .state
4030            .judgements
4031            .iter()
4032            .filter_map(|j| j.ranking.first().copied())
4033            .collect();
4034        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4035
4036        // A judge whose private vote failed still counted once, in the initial
4037        // ranking; using it beats discarding a whole seat.
4038        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4039        let mut cast: Vec<char> = Vec::new();
4040        for (i, j) in self.state.judgements.iter().enumerate() {
4041            let vote = self
4042                .state
4043                .votes
4044                .iter()
4045                .find(|v| v.judge == i + 1)
4046                .and_then(|v| v.vote)
4047                .or_else(|| j.ranking.first().copied());
4048            if let Some(v) = vote {
4049                *first_choice.entry(v).or_insert(0) += 1;
4050                cast.push(v);
4051            }
4052        }
4053
4054        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4055        for j in &self.state.judgements {
4056            let n = j.ranking.len();
4057            for (pos, label) in j.ranking.iter().enumerate() {
4058                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4059            }
4060        }
4061
4062        let best = first_choice.values().copied().max().unwrap_or(0);
4063        let mut leaders: Vec<char> = first_choice
4064            .iter()
4065            .filter(|(_, v)| **v == best)
4066            .map(|(k, _)| *k)
4067            .collect();
4068        let mut tie_break = None;
4069        if leaders.len() > 1 {
4070            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4071            let borda_leaders: Vec<char> = leaders
4072                .iter()
4073                .copied()
4074                .filter(|l| borda[l] == top_borda)
4075                .collect();
4076            tie_break = Some(if borda_leaders.len() == 1 {
4077                format!(
4078                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4079                    leaders.len()
4080                )
4081            } else {
4082                format!(
4083                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4084                    leaders.len()
4085                )
4086            });
4087            leaders = borda_leaders;
4088            leaders.sort_unstable();
4089        }
4090        let winner = *leaders
4091            .first()
4092            .or(viable.first())
4093            .context("no candidate to declare a winner from")?;
4094
4095        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4096        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4097        let deliberated = !self.state.deliberation.is_empty();
4098
4099        // Whose verdict is this? A rate-limited seat is absent even if it
4100        // ranked before the limit hit, so presence is measured against the
4101        // recorded losses, not just "did a ranking ever appear".
4102        let quota_seats: std::collections::BTreeSet<&str> =
4103            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4104        let mut present = 0usize;
4105        for (i, j) in self.state.judgements.iter().enumerate() {
4106            if quota_seats.contains(j.seat.as_str()) {
4107                continue;
4108            }
4109            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4110            let voted = self
4111                .state
4112                .votes
4113                .iter()
4114                .any(|v| v.judge == i + 1 && v.vote.is_some());
4115            if ranked || voted {
4116                present += 1;
4117            }
4118        }
4119        // Strict majority of the configured panel. A bare majority is real
4120        // signal we can act on, while a minority verdict must never stand in
4121        // for a healthy one. A one-candidate run needs no panel at all, and
4122        // `judges` stays `0` rather than the roster size a panel that never
4123        // sat would otherwise be credited with.
4124        let needs_quorum = viable.len() > 1;
4125        let judges_total = if needs_quorum {
4126            self.roles.judges.len()
4127        } else {
4128            0
4129        };
4130        let quorum = if needs_quorum {
4131            judges_total / 2 + 1
4132        } else {
4133            0
4134        };
4135        let met_quorum = !needs_quorum || present >= quorum;
4136        let uncontested = (!needs_quorum).then(|| {
4137            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4138        });
4139
4140        self.state.event(
4141            "tally",
4142            match &uncontested {
4143                Some(reason) => format!("winner {winner} — {reason}"),
4144                None => format!(
4145                    "winner {winner} — votes {} | initial {} | {} changed | \
4146                     {present}/{judges_total} judges{}",
4147                    first_choice
4148                        .iter()
4149                        .map(|(k, v)| format!("{k}:{v}"))
4150                        .collect::<Vec<_>>()
4151                        .join(" "),
4152                    if unanimous_initial {
4153                        "unanimous"
4154                    } else {
4155                        "split"
4156                    },
4157                    changed_votes,
4158                    if met_quorum {
4159                        String::new()
4160                    } else {
4161                        format!(" — below quorum ({quorum} required)")
4162                    },
4163                ),
4164            },
4165        );
4166        if !met_quorum {
4167            self.state.event(
4168                "stall",
4169                format!(
4170                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4171                     the run stops here, resumable"
4172                ),
4173            );
4174        }
4175        self.state.tally = Some(Tally {
4176            first_choice,
4177            borda,
4178            winner,
4179            rankings: tops.len(),
4180            unanimous_initial,
4181            deliberated,
4182            changed_votes,
4183            unanimous_final,
4184            tie_break,
4185            judges: judges_total,
4186            present,
4187            quorum,
4188            met_quorum,
4189            uncontested,
4190        });
4191        self.state.status = if met_quorum {
4192            RunStatus::Reviewing
4193        } else {
4194            RunStatus::Stalled
4195        };
4196        self.state.save()?;
4197        Ok(())
4198    }
4199
4200    // ------------------------------------------------------------- recover
4201
4202    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4203    /// resumed toward completion once the transient cause clears.
4204    ///
4205    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4206    /// it toward the quorum, which is exactly the set of seats whose absence
4207    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4208    /// not depend on which node happened to hit the limit), or an ordinary
4209    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4210    /// seat is never disturbed.
4211    ///
4212    /// A seat that now answers with a usable ranking is "recovered": its
4213    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4214    /// `tally` counts it present again), and its vote re-collected. A seat that
4215    /// still fails keeps its loss and stays absent.
4216    ///
4217    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4218    /// to review/gate/merge), `false` when it is still below quorum (the run
4219    /// stays `Stalled`, still resumable for a later retry).
4220    #[allow(clippy::too_many_lines)]
4221    async fn recover_stall(&mut self) -> Result<bool> {
4222        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4223        // agent files with `magi task add` name the run that paid for it. The
4224        // prompt overlay is cloned alongside it because the waves borrow it
4225        // while `self` is mutably borrowed by the node's own bookkeeping.
4226        let run_id = self.state.id.clone();
4227        let prompts = self.state.config.prompts.clone();
4228        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4229        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4230        // is what keeps a non-quota collapse — or a quota loss recorded at the
4231        // deliberate node — from being a permanent dead-end on `--resume`.
4232        let quota_seats: BTreeSet<&str> =
4233            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4234        let absent: Vec<String> = self
4235            .state
4236            .judgements
4237            .iter()
4238            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4239            .map(|j| j.seat.clone())
4240            .collect();
4241        if absent.is_empty() {
4242            return Ok(false);
4243        }
4244        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4245        if viable.len() <= 1 {
4246            return Ok(false);
4247        }
4248        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4249        let language = self.state.config.graph.language.clone();
4250        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4251        let sessions = self.state.config.graph.sessions;
4252        let artifacts = agent::artifacts_dir(&self.state.dir());
4253        let root = self.state.worktree_root();
4254        let base_short = short(&self.state.base_commit);
4255        let candidates: Vec<Candidate> = viable.clone();
4256
4257        // Map each absent seat key to its 0-based position in `roles.judges`.
4258        let mut positions: Vec<usize> = absent
4259            .iter()
4260            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4261            .collect();
4262        if positions.is_empty() {
4263            return Ok(false);
4264        }
4265        positions.sort_unstable();
4266        positions.dedup();
4267
4268        // Re-rank the lost seats, one blind prompt each.
4269        let mut judge_jobs = Vec::new();
4270        for &j in &positions {
4271            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4272            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4273            let seat_key = format!("judge-{}", j + 1);
4274            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4275            let seat = self.seat(&seat_key, &spec.id);
4276            judge_jobs.push(SeatJob {
4277                spec,
4278                seat,
4279                prompt: prompt::judge(
4280                    &self.state.instruction,
4281                    &views,
4282                    self.roles.judges.len(),
4283                    &base_short,
4284                    &language,
4285                ),
4286                cwd: root.join(seat_key),
4287                timeout,
4288                allow_write: false,
4289                sessions,
4290                artifacts: artifacts.clone(),
4291                stem: format!("judge-{}-recover", j + 1),
4292                handover: None,
4293            });
4294        }
4295
4296        let labels_for_check = labels.clone();
4297        let mut judge_losses = Vec::new();
4298        let retries = self.state.config.graph.retries;
4299        let cache = self.state.config.cache_dir();
4300        let ctx = WaveCtx {
4301            carry_seats: false,
4302            pauses: Some((&self.pause, &self.interrupt)),
4303            run: &run_id,
4304            node: "judge",
4305            prompts: &prompts,
4306            cache: cache.as_deref(),
4307            round: None,
4308        };
4309        let results = ask_json_wave::<Ranking>(
4310            judge_jobs,
4311            Arc::clone(&self.sem),
4312            retries,
4313            &self.roles.judge_roster,
4314            &ctx,
4315            &mut judge_losses,
4316            &mut self.state,
4317            &move |r: &Ranking| r.validate(&labels_for_check),
4318        )
4319        .await;
4320        if self.state.resource_block.is_some() {
4321            self.stop_for_resource()?;
4322            return Ok(false);
4323        }
4324
4325        // Refresh the judgement of every seat that ranked again.
4326        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4327        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4328            let agent_id = seat.agent.clone();
4329            self.state.seats.insert(seat.key.clone(), seat);
4330            let record = &mut self.state.judgements[j];
4331            match res {
4332                Ok((ranking, out)) => {
4333                    record.agent = agent_id;
4334                    record.ranking = ranking.normalized();
4335                    record.reasons = ranking.reasons;
4336                    record.strengths = ranking.strengths;
4337                    record.confidence = ranking.confidence;
4338                    record.failed = None;
4339                    record.duration_ms = out.duration_ms;
4340                    recovered.insert(j);
4341                    self.state.event(
4342                        "recover",
4343                        format!("judge {} ranked again after the limit", j + 1),
4344                    );
4345                }
4346                Err(e) => {
4347                    self.state
4348                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4349                }
4350            }
4351        }
4352
4353        // Re-ask the votes of the seats that recovered a ranking.
4354        let mut vote_jobs = Vec::new();
4355        let mut vote_pos: Vec<usize> = Vec::new();
4356        for &j in &recovered {
4357            let seat_key = format!("judge-{}", j + 1);
4358            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4359            let seat = self.seat(&seat_key, &spec.id);
4360            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4361            let text = if has_context(&spec, &seat, sessions) {
4362                prompt::final_vote(&labels, &language)
4363            } else {
4364                full.clone()
4365            };
4366            vote_jobs.push(SeatJob {
4367                spec,
4368                seat,
4369                prompt: text,
4370                cwd: root.join(seat_key),
4371                timeout,
4372                allow_write: false,
4373                sessions,
4374                artifacts: artifacts.clone(),
4375                stem: format!("vote-judge-{}-recover", j + 1),
4376                handover: Some(full),
4377            });
4378            vote_pos.push(j);
4379        }
4380        let allowed = labels.clone();
4381        let mut vote_losses = Vec::new();
4382        let vote_retries = self.state.config.graph.retries;
4383        let vote_cache = self.state.config.cache_dir();
4384        let ctx = WaveCtx {
4385            carry_seats: false,
4386            pauses: Some((&self.pause, &self.interrupt)),
4387            run: &run_id,
4388            node: "vote",
4389            prompts: &prompts,
4390            cache: vote_cache.as_deref(),
4391            round: None,
4392        };
4393        let votes = ask_json_wave::<FinalVote>(
4394            vote_jobs,
4395            Arc::clone(&self.sem),
4396            vote_retries,
4397            &self.roles.judge_roster,
4398            &ctx,
4399            &mut vote_losses,
4400            &mut self.state,
4401            &move |v: &FinalVote| match v.label() {
4402                Some(c) if allowed.contains(&c) => Ok(()),
4403                other => bail!("vote {other:?} is not one of {allowed:?}"),
4404            },
4405        )
4406        .await;
4407        if self.state.resource_block.is_some() {
4408            self.stop_for_resource()?;
4409            return Ok(false);
4410        }
4411        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4412            let agent_id = seat.agent.clone();
4413            self.state.seats.insert(seat.key.clone(), seat);
4414            match res {
4415                Ok((v, _)) => {
4416                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4417                        rec.vote = v.label();
4418                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4419                    } else {
4420                        self.state.votes.push(VoteRecord {
4421                            judge: j + 1,
4422                            agent: agent_id,
4423                            vote: v.label(),
4424                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4425                            changed: false,
4426                        });
4427                    }
4428                    self.state.event(
4429                        "recover",
4430                        format!("judge {} voted again after the limit", j + 1),
4431                    );
4432                }
4433                Err(e) => {
4434                    self.state
4435                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4436                }
4437            }
4438        }
4439
4440        // A seat that ranked again is present even if its re-vote failed —
4441        // `tally` falls back to the initial ranking's first choice — so clear
4442        // its quota loss. Seats that still fail keep theirs and stay absent.
4443        let recovered_keys: BTreeSet<String> = recovered
4444            .iter()
4445            .map(|&j| format!("judge-{}", j + 1))
4446            .collect();
4447        self.state
4448            .quota
4449            .retain(|q| !recovered_keys.contains(&q.seat));
4450        // A seat that hit the limit again is a fresh loss, not the old one:
4451        // replace the stale entry so the history stays one-per-seat and the
4452        // daemon can tell this attempt's loss from a previous session's.
4453        for loss in judge_losses.into_iter().chain(vote_losses) {
4454            if recovered_keys.contains(&loss.seat) {
4455                continue;
4456            }
4457            self.state.quota.retain(|q| q.seat != loss.seat);
4458            self.state.quota.push(loss);
4459        }
4460
4461        // Recompute the verdict from the refreshed panel.
4462        self.state.tally = None;
4463        self.tally()?;
4464        Ok(self
4465            .state
4466            .tally
4467            .as_ref()
4468            .map(|t| t.met_quorum)
4469            .unwrap_or(false))
4470    }
4471
4472    // ---------------------------------------------------------------- graft
4473
4474    /// Port the strengths the judges attributed to the losing candidates onto
4475    /// the winner, once, between the tally and the review loop.
4476    ///
4477    /// Best-effort by construction: it returns nothing and fails nothing. A
4478    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4479    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4480    /// the run carries on as if the phase were off. See [`graft`] for the
4481    /// decisions and `AGENTS.md` for the invariants.
4482    ///
4483    /// The one exception is a rollback that could not be confirmed: the
4484    /// rejected commit may still be the winner's tip, so the run is blocked
4485    /// (resumable; the record stays `Running`, which makes a resume retry the
4486    /// restore) instead of reviewing, gating and merging it.
4487    async fn graft(&mut self) -> Result<()> {
4488        let Err(e) = self.graft_inner().await else {
4489            return Ok(());
4490        };
4491        if e.is::<crate::repo_lock::ResourceBlocked>() {
4492            return Err(e);
4493        }
4494        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4495            self.state.status = RunStatus::Blocked;
4496            self.state.event("graft", format!("blocked: {e:#}"));
4497            self.state.save()?;
4498            return Err(e);
4499        }
4500        // An error after the `Running` record was saved may have left the
4501        // fixer's work in the tree: the record must not stay unsettled
4502        // (a later resume would reset past the review fixes), so confirm the
4503        // restore now, or block.
4504        if let Some(rec) = self.state.graft.clone()
4505            && !rec.outcome.settled()
4506        {
4507            if let Some(winner) = self.state.winner().cloned() {
4508                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4509                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4510                        return Err(re);
4511                    }
4512                    self.state.status = RunStatus::Blocked;
4513                    self.state.event("graft", format!("blocked: {re:#}"));
4514                    self.state.save()?;
4515                    return Err(re);
4516                }
4517            }
4518            if let Some(g) = self.state.graft.as_mut() {
4519                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4520            }
4521        }
4522        self.state.event("graft", format!("skipped: {e:#}"));
4523        if let Err(e) = self.state.save() {
4524            tracing::warn!("save after graft: {e:#}");
4525        }
4526        Ok(())
4527    }
4528
4529    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4530    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4531    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4532        let _coordination = self.coordinate(wt).await?;
4533        crate::recovery::restore_repair(wt, pre)
4534            .await
4535            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4536    }
4537
4538    /// A skip is a run event, recorded once however often the node is re-entered.
4539    fn graft_skipped(&mut self, why: &str) {
4540        let message = format!("skipped: {why}");
4541        if !self
4542            .state
4543            .events
4544            .iter()
4545            .any(|e| e.node == "graft" && e.message == message)
4546        {
4547            self.state.event("graft", message);
4548        }
4549    }
4550
4551    async fn graft_inner(&mut self) -> Result<()> {
4552        // A round that was started and never finished (crash, kill): the
4553        // fixer may have left anything in the tree, so put the winner back.
4554        if let Some(rec) = self.state.graft.clone()
4555            && !rec.outcome.settled()
4556        {
4557            // Reviews exist only if the graft ended without its record being
4558            // settled (a swallowed error): the tree has moved on since, and
4559            // a reset to `pre` would discard the review fixes.
4560            if self.state.reviews.is_empty()
4561                && let Some(winner) = self.state.winner().cloned()
4562            {
4563                self.graft_restore(&winner.worktree, &rec.pre).await?;
4564            }
4565            if rec.outcome == GraftOutcome::WaitingForRestore {
4566                self.state.graft = None;
4567                self.state.save()?;
4568            } else {
4569                if let Some(g) = self.state.graft.as_mut() {
4570                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4571                }
4572                self.state.event(
4573                    "graft",
4574                    "rolled back: the previous round was interrupted; the winner is untouched",
4575                );
4576                self.state.save()?;
4577                return Ok(());
4578            }
4579        }
4580        let plan = match graft::plan(&self.state) {
4581            Ok(plan) => plan,
4582            Err(skip) => {
4583                if skip != graft::Skip::AlreadyDone {
4584                    self.graft_skipped(skip.reason());
4585                    self.state.save()?;
4586                }
4587                return Ok(());
4588            }
4589        };
4590        let Some(winner) = self.state.winner().cloned() else {
4591            return Ok(());
4592        };
4593        let wt = winner.worktree.clone();
4594        if !git::is_clean(&wt).await? {
4595            self.graft_skipped("the winner's worktree has uncommitted changes");
4596            self.state.save()?;
4597            return Ok(());
4598        }
4599        let pre = git::rev_parse(&wt, "HEAD").await?;
4600        // Saved before the fixer is called: a stop from here on is repaired
4601        // by the branch above on the next entry.
4602        self.state.graft = Some(GraftRecord {
4603            pre: pre.clone(),
4604            post: None,
4605            lines: None,
4606            limit: None,
4607            agent: None,
4608            outcome: GraftOutcome::Running,
4609            requested: graft::requested(&plan),
4610            adoption: None,
4611            adoption_error: None,
4612        });
4613        self.state.event(
4614            "graft",
4615            format!(
4616                "porting strengths of candidate(s) {} onto {}",
4617                plan.strengths
4618                    .iter()
4619                    .map(|(l, _, _)| l.to_string())
4620                    .collect::<Vec<_>>()
4621                    .join(", "),
4622                plan.winner
4623            ),
4624        );
4625        self.state.save()?;
4626
4627        let fix_prompt = prompt::graft(
4628            &self.state.instruction,
4629            plan.winner,
4630            &plan.strengths,
4631            &self.state.config.graph.language,
4632        );
4633        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4634        let sessions = self.state.config.graph.sessions;
4635        let artifacts = agent::artifacts_dir(&self.state.dir());
4636        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4637        let (_, seat, out) = self
4638            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4639                SeatJob {
4640                    prompt: fix_prompt.clone(),
4641                    spec,
4642                    seat,
4643                    cwd: wt.clone(),
4644                    timeout,
4645                    allow_write: true,
4646                    sessions,
4647                    artifacts: artifacts.clone(),
4648                    stem: "graft".to_owned(),
4649                    handover: None,
4650                }
4651            })
4652            .await;
4653        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4654            // Preserve the need to restore earlier fallback edits without
4655            // confusing an unstarted seat with an interrupted invocation.
4656            self.state.graft.as_mut().expect("graft record").outcome =
4657                GraftOutcome::WaitingForRestore;
4658            return self.stop_for_resource();
4659        }
4660        let mut why: Option<String> = None;
4661        match &out {
4662            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4663                why = Some("the fixer produced nothing usable".to_owned());
4664            }
4665            AgentOutcome::Ok(_) => {}
4666            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4667            AgentOutcome::Quota(o) => {
4668                self.state.quota.push(QuotaLoss::from_report(
4669                    seat.key.clone(),
4670                    "graft",
4671                    o.quota.as_ref(),
4672                ));
4673                why = Some("rate limited (quota)".to_owned());
4674            }
4675            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4676                why = Some(format!(
4677                    "the fixer failed: {}",
4678                    e.lines()
4679                        .next()
4680                        .unwrap_or("")
4681                        .chars()
4682                        .take(160)
4683                        .collect::<String>()
4684                ));
4685            }
4686        }
4687        let agent_id = seat.agent.clone();
4688        self.state.seats.insert(seat.key.clone(), seat);
4689
4690        let mut post = None;
4691        let mut lines = None;
4692        let _coordination = self.coordinate(&wt).await?;
4693        if why.is_none() {
4694            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4695            if let Ok(r) = &rescued {
4696                self.state.note_withheld("graft", &r.withheld);
4697            }
4698            let head = git::rev_parse(&wt, "HEAD").await?;
4699            // What is measured must be what gets reviewed and gated: a failed
4700            // rescue, or anything still uncommitted (withheld lockfiles
4701            // aside, which are never part of the tree), rejects the graft.
4702            let dirty = match &rescued {
4703                Ok(r) => git::status_porcelain(&wt)
4704                    .await
4705                    .map(|st| {
4706                        st.lines().any(|l| {
4707                            !r.withheld
4708                                .iter()
4709                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4710                        })
4711                    })
4712                    .unwrap_or(true),
4713                Err(_) => true,
4714            };
4715            if dirty {
4716                why = Some("the fixer left changes that could not be committed".to_owned());
4717            } else if head == pre {
4718                why = Some("the fixer changed nothing".to_owned());
4719            } else {
4720                lines = git::changed_lines(&wt, &pre, &head).await;
4721                post = Some(head);
4722            }
4723        }
4724        let (adoption, adoption_error) = match &out {
4725            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4726            _ => (None, None),
4727        };
4728        let requested = graft::requested(&plan);
4729        let outcome = match why {
4730            None => {
4731                self.state.event(
4732                    "graft",
4733                    format!(
4734                        "applied: {}",
4735                        lines.map_or_else(
4736                            || "changed lines unavailable".to_owned(),
4737                            |n| format!("{n} changed lines")
4738                        )
4739                    ),
4740                );
4741                GraftOutcome::Applied
4742            }
4743            Some(why) => {
4744                // Not settled until the restore is confirmed: the record
4745                // stays `Running`, so a resume retries it.
4746                crate::recovery::restore_repair(&wt, &pre)
4747                    .await
4748                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4749                self.state.event(
4750                    "graft",
4751                    format!("rolled back: {why}; the winner is untouched"),
4752                );
4753                GraftOutcome::RolledBack(why)
4754            }
4755        };
4756        self.state.graft = Some(GraftRecord {
4757            pre,
4758            post,
4759            lines,
4760            limit: None,
4761            agent: Some(agent_id),
4762            outcome,
4763            requested,
4764            adoption,
4765            adoption_error,
4766        });
4767        self.state.save()?;
4768        Ok(())
4769    }
4770
4771    // ----------------------------------------------------------------- fold
4772
4773    async fn fold_losers(&mut self) -> Result<()> {
4774        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4775            return Ok(());
4776        };
4777        if !self.state.candidate_preservation.is_empty() {
4778            bail!("candidate preservation incomplete; refusing cleanup");
4779        }
4780        let repo = self.state.repo.clone();
4781        let _coordination = self.coordinate(&repo).await?;
4782        let mut folded = Vec::new();
4783        for i in 0..self.state.candidates.len() {
4784            let c = &self.state.candidates[i];
4785            if c.label == winner || c.folded {
4786                continue;
4787            }
4788            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4789            remove_preserved_worktree(&repo, &wt).await?;
4790            if git::branch_exists(&repo, &branch).await?
4791                && !git::branch_delete(&repo, &branch).await?
4792            {
4793                bail!("candidate branch removal refused");
4794            }
4795            self.state.candidates[i].folded = true;
4796            self.state.save()?;
4797            folded.push(label.to_string());
4798        }
4799        // The judges are finished; their checkouts are pure cost from here.
4800        let root = self.state.worktree_root();
4801        for j in 1..=self.roles.judges.len() {
4802            let wt = root.join(format!("judge-{j}"));
4803            if wt.exists() {
4804                remove_preserved_worktree(&repo, &wt).await?;
4805            }
4806        }
4807        // The design-deliberation stage is finished by the time a tally
4808        // exists — same reasoning as the judges above.
4809        if self.state.config.graph.advise {
4810            for k in 1..=self.state.config.graph.advisors {
4811                let wt = root.join(format!("advisor-{k}"));
4812                if wt.exists() {
4813                    remove_preserved_worktree(&repo, &wt).await?;
4814                }
4815            }
4816        }
4817        if !folded.is_empty() {
4818            self.state
4819                .event("fold", format!("folded candidates {}", folded.join(", ")));
4820            self.state.save()?;
4821        }
4822        Ok(())
4823    }
4824
4825    // ------------------------------------------------------------ base sync
4826
4827    async fn hold_base_sync(&mut self) -> Result<bool> {
4828        self.hold_base_sync_as(None).await
4829    }
4830
4831    async fn hold_base_sync_kind(
4832        &mut self,
4833        kind: crate::base_sync_recovery::StopKind,
4834    ) -> Result<()> {
4835        self.hold_base_sync_as(Some(kind)).await?;
4836        Ok(())
4837    }
4838
4839    async fn hold_base_sync_as(
4840        &mut self,
4841        kind: Option<crate::base_sync_recovery::StopKind>,
4842    ) -> Result<bool> {
4843        if !self.state.base_sync_blocked() {
4844            self.state.base_sync_recovery.stop = None;
4845            return Ok(false);
4846        }
4847        let head = match self.state.winner() {
4848            Some(w) => git::rev_parse(&w.worktree, "HEAD")
4849                .await
4850                .unwrap_or_default(),
4851            None => String::new(),
4852        };
4853        self.state.record_base_sync_stop(&head);
4854        if let Some(kind) = kind {
4855            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
4856        }
4857        self.state.save()?;
4858        Ok(true)
4859    }
4860
4861    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4862    /// anything verifies it.
4863    ///
4864    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4865    /// read whatever is checked out in the winner's worktree. Left alone that
4866    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4867    /// it when the run *branched* - and a run takes long enough that the base
4868    /// has usually moved by the time it gets here. A gate that ran there
4869    /// answers "green on the commit this run started from", not "green on
4870    /// what is about to land", and the difference showed up three times in
4871    /// one day as a green run whose merge would have reverted a file another
4872    /// pull request had already landed.
4873    ///
4874    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4875    /// `land::Step::Rebase` calls, rather than a second implementation of the
4876    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4877    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4878    /// doc warns about ("two answers to a question nobody notices until a
4879    /// diff is wrong").
4880    ///
4881    /// Foreign-checkout reviews instead merge the base into the owned shadow
4882    /// worktree, preserving the ancestry required for fast-forward delivery.
4883    /// A conflicting merge is aborted and held for operator resolution.
4884    ///
4885    /// A conflict is not the end of the road: the standing rebase is handed
4886    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4887    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4888    /// land). Once it finishes, review and the gate run as usual on the
4889    /// rebased tree, which is where a breakage the new base caused is caught
4890    /// by the ordinary gate-fix round. magi resolves nothing itself.
4891    ///
4892    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4893    /// `state.base_sync.attempts`, is how many times the base is *rebased
4894    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4895    /// *conflict* was given to a fixer. When the fixer cannot finish the
4896    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4897    /// what was tried (rounds spent, paths still conflicted) and the branch
4898    /// and worktree stay exactly as they were - untouched, for a person to
4899    /// look at - which is also what makes re-entering this function
4900    /// afterwards a no-op instead of a second attempt at the same wall. A
4901    /// push failure ends the same way.
4902    async fn sync_to_base(&mut self) -> Result<()> {
4903        if self.state.status == RunStatus::AlreadyInBase {
4904            return Ok(());
4905        }
4906        self.hold_base_sync().await?;
4907        let conflicted = self
4908            .state
4909            .base_sync
4910            .as_ref()
4911            .is_some_and(|s| s.conflict.is_some());
4912        let Some(winner) = self.state.winner().cloned() else {
4913            return Ok(());
4914        };
4915
4916        let repo = self.state.repo.clone();
4917        let remote = self.state.config.merge.remote.clone();
4918        let base_branch = self.state.base_branch.clone();
4919        let tracking = format!("{remote}/{base_branch}");
4920
4921        let fetched_base =
4922            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
4923        // No network, or the remote never had this branch: the run already
4924        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4925        // otherwise), and one that got this far is not blocked by a fetch
4926        // that fails now. It just has no newer tip to compare against.
4927        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4928            return Ok(());
4929        };
4930
4931        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4932        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4933        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4934
4935        // Before any rebase, and before a recorded conflict is honoured: a
4936        // branch whose change reached the base under other commit ids has
4937        // nothing to rebase and nothing to conflict with, and a run that
4938        // already stopped on that phantom conflict recovers here on resume.
4939        // `behind == 0` with head == tip is a branch the base has since taken
4940        // in whole, whether or not a conflict was ever recorded: the ancestry
4941        // proof must still run (`classify` ignores a head still on the start
4942        // commit).
4943        if (behind > 0 || conflicted || head == tip)
4944            && self
4945                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4946                .await?
4947        {
4948            return Ok(());
4949        }
4950        if conflicted && self.state.base_recovery_pending() {
4951            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
4952            let owner =
4953                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
4954            if !fetched_base
4955                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
4956                || !git::is_clean(&winner.worktree).await?
4957                || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
4958            {
4959                return Ok(());
4960            }
4961        }
4962        let recovery = conflicted && self.state.consume_base_recovery(&head);
4963        if conflicted {
4964            // Consumption (including a refused changed HEAD) is durable before Git.
4965            self.state.save()?;
4966            if !recovery {
4967                return Ok(());
4968            }
4969        }
4970
4971        if behind == 0 {
4972            // A fixer-finished rebase moves the branch ref before the
4973            // winner's worktree is told (`sync_to_head` below). A run that
4974            // died in between resumes here with `behind == 0` and a tree still
4975            // holding the pre-rebase files, which review and the gate would
4976            // then read. That state is exactly: HEAD moved off the tip the
4977            // rebase started from, yet the tree is still identical to that
4978            // tip. A tree with edits of its own differs from it, so nothing
4979            // is thrown away.
4980            if let Some(from) = self
4981                .state
4982                .rebase_fixes
4983                .iter()
4984                .rev()
4985                .find_map(|r| r.from.clone())
4986                && from != head
4987                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4988                    .await
4989                    .is_ok_and(|o| o.ok())
4990            {
4991                git::sync_to_head(&winner.worktree).await?;
4992            }
4993            // An earlier attempt may have rebased the branch locally and died
4994            // before pushing it (only the fresh-rebase arm below pushes).
4995            // Publish it now, so the plain push at PR time is not refused as
4996            // a non-fast-forward. A run already holding a recorded conflict
4997            // never reaches here; that case is out of scope.
4998            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4999            if let Some(why) = &conflict {
5000                self.state.status = RunStatus::Blocked;
5001                self.state.event("land", why.clone());
5002            }
5003            self.state.base_sync = Some(BaseSync {
5004                tip,
5005                behind: 0,
5006                attempts: attempts + usize::from(recovery),
5007                conflict,
5008                already_in: None,
5009            });
5010            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5011                .await?;
5012            self.state.save()?;
5013            return Ok(());
5014        }
5015
5016        if attempts >= BASE_SYNC_ROUNDS && !recovery {
5017            let why = format!(
5018                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5019                 synchronization(s); synchronizing again would only race it",
5020                winner.branch
5021            );
5022            self.state.status = RunStatus::Blocked;
5023            self.state.base_sync = Some(BaseSync {
5024                tip,
5025                behind,
5026                attempts,
5027                conflict: Some(why.clone()),
5028                already_in: None,
5029            });
5030            self.state.event("land", why);
5031            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5032                .await?;
5033            self.state.save()?;
5034            return Ok(());
5035        }
5036
5037        if self.state.foreign_checkout.is_some() {
5038            // Delivery to the operator's checkout is fast-forward only. A
5039            // rebase here would discard its saved head from our ancestry.
5040            let wait = repository_wait(&self.state, None);
5041            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5042            let merged = git::git_raw(
5043                &winner.worktree,
5044                &[
5045                    "-c",
5046                    "core.hooksPath=/dev/null",
5047                    "merge",
5048                    "--no-ff",
5049                    "--no-autostash",
5050                    "--no-edit",
5051                    "-m",
5052                    &format!("Merge {tracking} before verifying"),
5053                    &tip,
5054                ],
5055            )
5056            .await?;
5057            let conflict = if merged.ok() {
5058                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5059                self.state.event(
5060                    "land",
5061                    format!("merged {tracking} into {} before verifying", winner.branch),
5062                );
5063                None
5064            } else {
5065                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5066                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5067                }
5068                let why = format!(
5069                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5070                    winner.branch,
5071                    merged.stderr.chars().take(600).collect::<String>()
5072                );
5073                self.state.status = RunStatus::Blocked;
5074                self.state.event("land", &why);
5075                Some(why)
5076            };
5077            let synced = conflict.is_none();
5078            self.state.base_sync = Some(BaseSync {
5079                tip,
5080                behind: if synced { 0 } else { behind },
5081                attempts: attempts + 1,
5082                conflict,
5083                already_in: None,
5084            });
5085            self.state.save()?;
5086            return Ok(());
5087        }
5088
5089        self.state.event(
5090            "land",
5091            format!(
5092                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5093                winner.branch
5094            ),
5095        );
5096        self.state.save()?;
5097
5098        // The remote's copy of the branch, read now and only if the fetch
5099        // really succeeded (a stale tracking ref must never pin a lease). It is
5100        // pushed over after a rebase only when it is a commit this branch
5101        // already contains, by ancestry or by patch (an earlier rebase of ours
5102        // that never reached the remote): anything else is somebody else's work.
5103        let branch_tracking = format!("{remote}/{}", winner.branch);
5104        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5105        let remote_tip = if self.state.task_pr_branch().is_none()
5106            && matches!(&fetched_branch, Ok(o) if o.ok())
5107        {
5108            git::rev_parse(&repo, &branch_tracking).await.ok()
5109        } else {
5110            None
5111        };
5112        // A remote tip this branch does not contain is somebody else's work:
5113        // rebasing would leave a local tip that can never be pushed. Stop
5114        // before touching anything and say so.
5115        if let Some(theirs) = &remote_tip
5116            && !git::is_ancestor(&repo, theirs, &head).await
5117            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5118                .await
5119                .is_ok_and(|missing| missing.is_empty())
5120        {
5121            let why = format!(
5122                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5123                 them",
5124                short(theirs),
5125                winner.branch
5126            );
5127            self.state.status = RunStatus::Blocked;
5128            self.state.base_sync = Some(BaseSync {
5129                tip,
5130                behind,
5131                attempts: attempts + usize::from(recovery),
5132                conflict: Some(why.clone()),
5133                already_in: None,
5134            });
5135            self.state.event("land", why);
5136            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5137                .await?;
5138            self.state.save()?;
5139            return Ok(());
5140        }
5141
5142        let scratch = self.state.dir().join("base-sync");
5143        let rebased =
5144            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5145                .await
5146            {
5147                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5148                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5149                Err(e) => Err(e),
5150            };
5151        let attempts = attempts + 1;
5152        let stop_kind = match &rebased {
5153            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5154            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5155            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5156        };
5157        match rebased {
5158            Ok(None) => {
5159                // The branch ref moved, but a worktree that already had it
5160                // checked out (the winner's) was not told; sync its index and
5161                // files before anything reads them.
5162                git::sync_to_head(&winner.worktree).await?;
5163                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5164                let mut conflict = None;
5165                if let Some(pinned) = &remote_tip {
5166                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5167                    match pushed {
5168                        Ok(o) if o.ok() => self.state.event(
5169                            "land",
5170                            format!("pushed rebased {} to {remote}", winner.branch),
5171                        ),
5172                        Ok(o) => {
5173                            conflict = Some(format!(
5174                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5175                                winner.branch,
5176                                short(pinned),
5177                                o.stderr.chars().take(600).collect::<String>()
5178                            ));
5179                        }
5180                        Err(e) => {
5181                            conflict = Some(format!(
5182                                "rebased {} locally but could not push it: {e:#}",
5183                                winner.branch
5184                            ));
5185                        }
5186                    }
5187                }
5188                if let Some(why) = &conflict {
5189                    self.state.status = RunStatus::Blocked;
5190                    self.state.event("land", why.clone());
5191                }
5192                self.state.base_sync = Some(BaseSync {
5193                    tip: tip.clone(),
5194                    behind: 0,
5195                    attempts,
5196                    conflict,
5197                    already_in: None,
5198                });
5199                self.state
5200                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
5201            }
5202            Ok(Some(conflict)) => {
5203                let why = format!(
5204                    "{} conflicts with {tracking} and did not rebase: {}",
5205                    winner.branch,
5206                    conflict.chars().take(600).collect::<String>()
5207                );
5208                self.state.status = RunStatus::Blocked;
5209                self.state.base_sync = Some(BaseSync {
5210                    tip,
5211                    behind,
5212                    attempts,
5213                    conflict: Some(why.clone()),
5214                    already_in: None,
5215                });
5216                self.state.event("land", why);
5217            }
5218            // A refused Git write or a busy repository: nothing is wrong with
5219            // the branch. The scratch rebase stays standing and the run
5220            // resumes into it (`base_sync` must not claim a conflict).
5221            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5222                self.state.save()?;
5223                return Ok(());
5224            }
5225            Err(e) => {
5226                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5227                self.state.status = RunStatus::Blocked;
5228                self.state.base_sync = Some(BaseSync {
5229                    tip,
5230                    behind,
5231                    attempts,
5232                    conflict: Some(why.clone()),
5233                    already_in: None,
5234                });
5235                self.state.event("land", why);
5236            }
5237        }
5238        if !self.state.base_sync_blocked() {
5239            self.state.base_sync_recovery.stop = None;
5240        }
5241        self.hold_base_sync_kind(stop_kind).await?;
5242        self.state.save()?;
5243        Ok(())
5244    }
5245
5246    /// Push a branch an earlier attempt rebased locally but never published,
5247    /// pinned to the remote tip read right after a successful fetch. Returns
5248    /// the reason when the run must stop; `None` when there was nothing to do
5249    /// (no remote copy, the same tip, or a remote copy this branch already
5250    /// contains, which the PR-time push fast-forwards) or the push succeeded.
5251    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5252        // Task competitions publish only after the PR text gate, to their
5253        // public head; never resurrect a candidate branch on the remote.
5254        if self.state.task_pr_branch().is_some() {
5255            return None;
5256        }
5257        let repo = self.state.repo.clone();
5258        let remote = self.state.config.merge.remote.clone();
5259        let fetched = git::fetch(&repo, &remote, branch).await;
5260        if !matches!(&fetched, Ok(o) if o.ok()) {
5261            return None;
5262        }
5263        let branch_tracking = format!("{remote}/{branch}");
5264        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5265        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5266            return None;
5267        }
5268        if !crate::reconcile::origin_missing(&repo, head, &theirs)
5269            .await
5270            .is_ok_and(|missing| missing.is_empty())
5271        {
5272            return Some(format!(
5273                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5274                 them",
5275                short(&theirs)
5276            ));
5277        }
5278        match git::push_pinned(&repo, &remote, branch, &theirs).await {
5279            Ok(o) if o.ok() => {
5280                self.state
5281                    .event("land", format!("pushed rebased {branch} to {remote}"));
5282                None
5283            }
5284            Ok(o) => Some(format!(
5285                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5286                 someone may have pushed): {}",
5287                short(&theirs),
5288                o.stderr.chars().take(600).collect::<String>()
5289            )),
5290            Err(e) => Some(format!(
5291                "{branch} is rebased locally but could not be pushed: {e:#}"
5292            )),
5293        }
5294    }
5295
5296    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
5297    /// is already on `tip` under other commit ids ([`crate::already`]); returns
5298    /// whether it did.
5299    ///
5300    /// Checked only when the base is ahead of the branch. A failing check is
5301    /// "not proven" - the ordinary rebase path then decides - never a reason to
5302    /// stop the run.
5303    ///
5304    /// The remote copy of the branch is held to the same standard as the local
5305    /// one: if it carries a tip this worktree does not, that tip must itself be
5306    /// proven in the base, or nothing is settled (a pull request would
5307    /// otherwise be closed over commits nobody checked). The pull request is
5308    /// closed *before* the terminal status is saved; if that fails for a
5309    /// reason other than a refusal (no network, a `gh` error) the run is left
5310    /// `Blocked` with the reason as its conflict, which a resume retries -
5311    /// the same recovery a phantom conflict gets.
5312    async fn settle_already_in(
5313        &mut self,
5314        branch: &str,
5315        tip: &str,
5316        head: &str,
5317        attempts: usize,
5318        behind: usize,
5319    ) -> Result<bool> {
5320        let repo = self.state.repo.clone();
5321        let remote = self.state.config.merge.remote.clone();
5322        let start = self.state.base_commit.clone();
5323        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5324            Ok(Some(e)) => e,
5325            Ok(None) => return Ok(false),
5326            Err(e) => {
5327                tracing::warn!("already-in-base check for {branch}: {e:#}");
5328                return Ok(false);
5329            }
5330        };
5331        let mut verified = vec![head.to_owned()];
5332        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5333        let fetched = git::fetch(&repo, &remote, &public_branch).await;
5334        if matches!(&fetched, Ok(o) if o.ok())
5335            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5336            && theirs != head
5337        {
5338            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5339                Ok(Some(_)) => verified.push(theirs),
5340                _ => return Ok(false),
5341            }
5342        }
5343        let base_branch = self.state.base_branch.clone();
5344        let message = format!(
5345            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5346             land",
5347            evidence.names(),
5348            evidence.proof.as_str()
5349        );
5350        let closed =
5351            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5352                .await;
5353        match closed {
5354            Ok(Ok(url)) => self
5355                .state
5356                .event("land", format!("closed {url}: superseded on {base_branch}")),
5357            Ok(Err(why)) => self
5358                .state
5359                .event("land", format!("did not close a pull request: {why}")),
5360            Err(e) => {
5361                let why = format!(
5362                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
5363                     not be closed ({e:#}); resume to retry"
5364                );
5365                self.state.status = RunStatus::Blocked;
5366                self.state.base_sync = Some(BaseSync {
5367                    tip: tip.to_owned(),
5368                    behind,
5369                    attempts,
5370                    conflict: Some(why.clone()),
5371                    already_in: None,
5372                });
5373                self.state.event("land", why);
5374                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5375                    .await?;
5376                self.state.save()?;
5377                return Ok(true);
5378            }
5379        }
5380        self.state.status = RunStatus::AlreadyInBase;
5381        self.state.base_sync_recovery.stop = None;
5382        self.state.base_sync = Some(BaseSync {
5383            tip: tip.to_owned(),
5384            behind,
5385            attempts,
5386            conflict: None,
5387            already_in: Some(evidence),
5388        });
5389        self.state.event("land", message);
5390        self.state.save()?;
5391        self.settle_questions();
5392        Ok(true)
5393    }
5394
5395    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
5396    /// last landed the winner on, once it has run, else the commit the run
5397    /// branched from.
5398    ///
5399    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
5400    /// and `vote` all happen before there is a winner to rebase, so they
5401    /// compare every candidate against the branch point on purpose, and a
5402    /// base that moves after they are already done cannot change an answer
5403    /// they already gave.
5404    fn landing_base(&self) -> String {
5405        self.state
5406            .base_sync
5407            .as_ref()
5408            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5409    }
5410
5411    // ------------------------------------------------------- operator fix
5412
5413    /// Route specific, already-recorded review findings to a fixer for a
5414    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5415    /// entry point.
5416    ///
5417    /// Distinct from `review_loop`'s own fix step in three ways: it never
5418    /// runs a reviewer wave, it never spends review-round budget, and what
5419    /// happened is recorded as an [`OperatorFixRequest`] appended to
5420    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5421    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5422    /// and vote must never be rewritten to look like a manufactured blocking
5423    /// verdict.
5424    ///
5425    /// Only meaningful once review has actually concluded: `Ready` (handed
5426    /// off with findings still open, or simply concluded clean while minor
5427    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5428    /// gate failed). Everything else is refused: a run still in progress
5429    /// should simply be resumed, and a `Merged` run's branch has already
5430    /// landed — reopening *this* run's own record cannot change that, so the
5431    /// answer there is a fresh `magi review <branch>`.
5432    ///
5433    /// A real commit here re-verifies through a fresh, ordinary review-only
5434    /// run on the same branch ([`Self::review`]) rather than reopening this
5435    /// run's own `review_loop`: once any round in this run's history went
5436    /// clean, `review_conclusion` treats that as permanent by design (the
5437    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5438    /// way to force one more genuine reviewer wave out of *this* run without
5439    /// either rewriting history or weakening that guarantee for every other
5440    /// caller. A review-only run costs nothing extra — no implementation, no
5441    /// judging, no vote — and exercises the exact same review → verify →
5442    /// gate → (human) merge path, unmodified.
5443    pub async fn fix_selected(
5444        &mut self,
5445        ids: &[String],
5446        reason: &str,
5447        allow_stale: bool,
5448    ) -> Result<()> {
5449        let reason = reason.trim();
5450        if reason.is_empty() {
5451            bail!("a fix request needs a reason — that is the operator's own record of why");
5452        }
5453        if ids.is_empty() {
5454            bail!("no finding id given");
5455        }
5456        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5457            bail!(
5458                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5459                 has already concluded — can be given a targeted fix. A run still \
5460                 in progress should simply be resumed; a `merged` run's branch has \
5461                 already landed, so its answer is a fresh `magi review <branch>`, \
5462                 not reopening this run's own record",
5463                self.state.id,
5464                self.state.status.as_str()
5465            );
5466        }
5467        let Some(winner) = self.state.winner().cloned() else {
5468            bail!("run {} has no winning candidate to fix", self.state.id);
5469        };
5470        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5471            bail!(
5472                "branch `{}` no longer exists; this run cannot be extended",
5473                winner.branch
5474            );
5475        }
5476        let home = crate::run::home();
5477        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5478            bail!(
5479                "run {} is currently being worked on by another magi process",
5480                self.state.id
5481            );
5482        }
5483        // Held for the rest of this call, including the follow-up review
5484        // below: two `magi fix` invocations against the same run must not
5485        // both reach the worktree manipulation further down, which would
5486        // otherwise race to remove and recreate the same directory — see
5487        // [`FixClaim`]'s own doc.
5488        let _claim = FixClaim::acquire(&self.state.dir())?;
5489
5490        // Resolve every id before spending anything — an unknown id refuses
5491        // the whole request rather than silently dropping it — and dedup
5492        // while keeping the operator's own order.
5493        let mut seen = BTreeSet::new();
5494        let mut findings = Vec::new();
5495        let mut missing = Vec::new();
5496        for id in ids {
5497            if !seen.insert(id.clone()) {
5498                continue;
5499            }
5500            match self.state.finding(id) {
5501                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5502                    id: f.id.clone(),
5503                    severity: f.severity,
5504                    reviewer_vote: rec.vote,
5505                    round: round.round,
5506                    round_head: round.head.clone(),
5507                    reviewer: rec.reviewer,
5508                    agent: rec.agent.clone(),
5509                    file: f.file.clone(),
5510                    line: f.line,
5511                    title: f.title.clone(),
5512                    detail: f.detail.clone(),
5513                    outcome: OperatorFixOutcome::Pending,
5514                }),
5515                None => missing.push(id.clone()),
5516            }
5517        }
5518        if !missing.is_empty() {
5519            bail!(
5520                "unknown finding id(s): {}; nothing was changed",
5521                missing.join(", ")
5522            );
5523        }
5524
5525        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5526        let stale_details: Vec<(String, String)> = findings
5527            .iter()
5528            .filter(|f| f.round_head != head_at_request)
5529            .map(|f| (f.id.clone(), f.round_head.clone()))
5530            .collect();
5531        let stale = !stale_details.is_empty();
5532        if stale && !allow_stale {
5533            bail!(
5534                "the branch has moved since some finding(s) were raised — {} — now \
5535                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5536                stale_details
5537                    .iter()
5538                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5539                    .collect::<Vec<_>>()
5540                    .join(", "),
5541                short(&head_at_request)
5542            );
5543        }
5544
5545        let request = OperatorFixRequest {
5546            requested_at: Timestamp::now(),
5547            reason: reason.to_owned(),
5548            findings,
5549            head_at_request: head_at_request.clone(),
5550            allow_stale,
5551            stale,
5552            fix: None,
5553            result_head: None,
5554            follow_up_review_run: None,
5555        };
5556        self.state.event(
5557            "fix",
5558            format!(
5559                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5560                request.findings.len(),
5561                request
5562                    .findings
5563                    .iter()
5564                    .map(|f| f.id.as_str())
5565                    .collect::<Vec<_>>()
5566                    .join(", "),
5567            ),
5568        );
5569        // Recorded now, before any worktree work or the fixer call itself —
5570        // and re-saved at each checkpoint below: a crash at any point after
5571        // this (mid fixer call, mid follow-up review) must not lose the fact
5572        // that this was requested, for which findings, and why. Everything
5573        // past this point reads and writes through `request_index` rather
5574        // than a local variable, since `request` itself is moved here.
5575        self.state.operator_fixes.push(request);
5576        self.state.save()?;
5577        let request_index = self.state.operator_fixes.len() - 1;
5578
5579        // A fresh, dedicated worktree for this one call, never the winner's
5580        // own worktree in place: that one may already be gone (folded away),
5581        // and reusing it in place would leave the branch checked out there
5582        // when the follow-up review below tries to check it out again. Freed
5583        // immediately after, either way — but only once confirmed clean:
5584        // `worktree_remove` is a `git worktree remove --force`, which would
5585        // otherwise discard uncommitted work left there by the operator or
5586        // another process before this had a chance to even look at it.
5587        let repo = self.state.repo.clone();
5588        let coordination = self.coordinate(&repo).await?;
5589        if winner.worktree.exists() {
5590            // Lockfiles a rescue commit withheld stay untracked on purpose and
5591            // are already recorded; they are not the operator's work to protect.
5592            let dirty = git::git(
5593                &winner.worktree,
5594                &["status", "--porcelain", "--untracked-files=all"],
5595            )
5596            .await?;
5597            let only_withheld = dirty.lines().all(|l| {
5598                l.strip_prefix("?? ")
5599                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5600            });
5601            if !only_withheld {
5602                bail!(
5603                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5604                     discard them first",
5605                    winner.worktree.display()
5606                );
5607            }
5608            git::worktree_remove(&self.state.repo, &winner.worktree)
5609                .await
5610                .ok();
5611        }
5612        let fix_worktree = self.state.worktree_root().join("operator-fix");
5613        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5614        git::git(
5615            &self.state.repo,
5616            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5617        )
5618        .await
5619        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5620        if !git::is_clean(&fix_worktree).await? {
5621            git::worktree_remove(&self.state.repo, &fix_worktree)
5622                .await
5623                .ok();
5624            bail!(
5625                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5626                winner.branch
5627            );
5628        }
5629        let repo_for_setup = self.state.repo.clone();
5630        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5631            Ok(()) => {
5632                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5633            }
5634            Err(e) => Err(e),
5635        };
5636        if let Err(e) = setup {
5637            git::worktree_remove(&self.state.repo, &fix_worktree)
5638                .await
5639                .ok();
5640            return Err(e);
5641        }
5642
5643        // This explicit operator request retries availability independently
5644        // of execute(); the previous call has already saved and cleaned up.
5645        self.state.resource_block = None;
5646
5647        let run_id = self.state.id.clone();
5648        let prompts = self.state.config.prompts.clone();
5649        let language = self.state.config.graph.language.clone();
5650        let sessions = self.state.config.graph.sessions;
5651        let artifacts = agent::artifacts_dir(&self.state.dir());
5652        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5653            .findings
5654            .iter()
5655            .map(|f| Finding {
5656                id: f.id.clone(),
5657                severity: f.severity,
5658                file: f.file.clone(),
5659                line: f.line,
5660                title: f.title.clone(),
5661                detail: f.detail.clone(),
5662            })
5663            .collect();
5664        let fix_prompt = prompt::operator_fix_in(
5665            &self.state.instruction,
5666            &finding_list,
5667            reason,
5668            &stale_details,
5669            &head_at_request,
5670            &language,
5671            &self.state.config.graph.github_language,
5672        );
5673        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5674        drop(coordination);
5675        let (job, seat, out) = self
5676            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5677                prompt: fix_prompt.clone(),
5678                spec,
5679                seat,
5680                cwd: fix_worktree.clone(),
5681                timeout,
5682                allow_write: true,
5683                sessions,
5684                artifacts: artifacts.clone(),
5685                stem: "operator-fix".to_owned(),
5686                handover: None,
5687            })
5688            .await;
5689        let agent_id = seat.agent.clone();
5690
5691        let mut fix = FixRecord {
5692            agent: agent_id,
5693            addressed: Vec::new(),
5694            rejected: Vec::new(),
5695            notes: String::new(),
5696            committed: false,
5697            failed: None,
5698            duration_ms: 0,
5699            continuation: None,
5700        };
5701        let mut final_seat = seat.clone();
5702        match out {
5703            AgentOutcome::Ok(o) => {
5704                fix.duration_ms = o.duration_ms;
5705                let parsed = verdict::extract_json::<FixReport>(&o.text);
5706                let incomplete_reason = match &parsed {
5707                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5708                        "the reply parsed, but it reported a command whose own CLI \
5709                         never confirmed an exit status"
5710                            .to_owned(),
5711                    ),
5712                    Ok(_) => None,
5713                    Err(e) => Some(e.to_string()),
5714                };
5715                match incomplete_reason {
5716                    None => {
5717                        let report = parsed.expect("checked Ok above");
5718                        fix.addressed = report.addressed;
5719                        fix.rejected = report.rejected;
5720                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5721                    }
5722                    Some(reason) => {
5723                        let (resumed_seat, resolved, failure, cont) = self
5724                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5725                            .await;
5726                        fix.duration_ms += cont.cumulative_wait_ms;
5727                        fix.continuation = Some(cont);
5728                        final_seat = resumed_seat;
5729                        match resolved {
5730                            Some(report) => {
5731                                fix.addressed = report.addressed;
5732                                fix.rejected = report.rejected;
5733                                fix.notes =
5734                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5735                            }
5736                            None => fix.failed = failure,
5737                        }
5738                    }
5739                }
5740            }
5741            AgentOutcome::Dropped(o) => {
5742                fix.duration_ms = o.duration_ms;
5743                let why = o
5744                    .dropped
5745                    .as_ref()
5746                    .map(|d| d.why.as_str())
5747                    .unwrap_or("the CLI ended the stream without delivering its answer");
5748                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5749            }
5750            AgentOutcome::Quota(o) => {
5751                self.state.quota.push(QuotaLoss::from_report(
5752                    final_seat.key.clone(),
5753                    "fix",
5754                    o.quota.as_ref(),
5755                ));
5756                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5757            }
5758            AgentOutcome::Failed(e) => fix.failed = Some(e),
5759            AgentOutcome::ResourceBlocked(e) => {
5760                fix.failed = Some(format!("the fixer was not started: {e}"));
5761            }
5762        }
5763        // Even a deferred call must rescue edits, release the worktree and
5764        // finalize its record below so an explicit `magi fix` can retry.
5765        if fix.continuation.is_none() {
5766            fix.continuation = Some(ContinuationRecord::not_needed());
5767        }
5768        self.state.seats.insert(final_seat.key.clone(), final_seat);
5769
5770        let rescue_message = format!(
5771            "magi: operator-selected fix ({}) (uncommitted work)",
5772            self.state.operator_fixes[request_index]
5773                .findings
5774                .iter()
5775                .map(|f| f.id.as_str())
5776                .collect::<Vec<_>>()
5777                .join(", ")
5778        );
5779        let repo = self.state.repo.clone();
5780        let coordination = self.coordinate(&repo).await?;
5781        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5782            self.state.note_withheld("fix", &r.withheld);
5783        }
5784        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5785        fix.committed = after != head_at_request;
5786        git::worktree_remove(&self.state.repo, &fix_worktree)
5787            .await
5788            .ok();
5789        drop(coordination);
5790
5791        self.state.event(
5792            "fix",
5793            match &fix.failed {
5794                Some(reason) => format!(
5795                    "operator fix: adoption report was lost ({reason}); {}",
5796                    if fix.committed {
5797                        "committed"
5798                    } else {
5799                        "NO new commit"
5800                    }
5801                ),
5802                None => format!(
5803                    "operator fix: {} addressed, {} rejected, {}",
5804                    fix.addressed.len(),
5805                    fix.rejected.len(),
5806                    if fix.committed {
5807                        "committed"
5808                    } else {
5809                        "NO new commit"
5810                    }
5811                ),
5812            },
5813        );
5814
5815        // Every selected finding gets an outcome — never left `Pending` once
5816        // the fixer's own turn is over. A report that never came back at all
5817        // marks every one of them `Unreported`, not silently "not addressed":
5818        // quota, a dropped stream, or an exhausted continuation are gaps in
5819        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5820        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5821        for f in &mut self.state.operator_fixes[request_index].findings {
5822            f.outcome = if fix.failed.is_some() {
5823                OperatorFixOutcome::Unreported
5824            } else if fix.addressed.contains(&f.id) {
5825                OperatorFixOutcome::Addressed
5826            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5827                OperatorFixOutcome::Rejected { why: r.why.clone() }
5828            } else {
5829                OperatorFixOutcome::Unreported
5830            };
5831        }
5832
5833        let committed = fix.committed;
5834        if committed {
5835            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5836        }
5837        self.state.operator_fixes[request_index].fix = Some(fix);
5838        // Saved again now that the fixer's own outcome is final, on top of
5839        // the save right after the request was first pushed above.
5840        self.state.save()?;
5841        if self.state.resource_block.is_some() {
5842            return self.stop_for_resource();
5843        }
5844
5845        if committed {
5846            self.state.event(
5847                "fix",
5848                format!(
5849                    "operator fix committed {}; opening a follow-up review-only run",
5850                    short(&after)
5851                ),
5852            );
5853            // The operator asked for the fix, and the follow-up serves whatever
5854            // task the run it follows served.
5855            let origin =
5856                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5857            match Self::review(
5858                &self.state.repo,
5859                &winner.branch,
5860                self.state.config.clone(),
5861                origin,
5862            )
5863            .await
5864            {
5865                Ok(mut follow_up) => {
5866                    follow_up.state.event(
5867                        "start",
5868                        format!(
5869                            "requested by an operator fix on run {} for finding(s) {}",
5870                            self.state.id,
5871                            self.state.operator_fixes[request_index]
5872                                .findings
5873                                .iter()
5874                                .map(|f| f.id.as_str())
5875                                .collect::<Vec<_>>()
5876                                .join(", "),
5877                        ),
5878                    );
5879                    follow_up.state.save()?;
5880                    let follow_up_id = follow_up.state.id.clone();
5881                    // The follow-up is a run like any other: it belongs to the
5882                    // task of the run it follows, or to one filed for it.
5883                    let adopted = match crate::direct::adopt(
5884                        &follow_up.state,
5885                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5886                    ) {
5887                        Ok(a) => a,
5888                        Err(e) => {
5889                            // An ownerless run must not spend agent calls; it
5890                            // stays saved, and `magi run --resume` adopts it.
5891                            self.state.event(
5892                                "fix",
5893                                format!(
5894                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5895                                ),
5896                            );
5897                            self.state.save()?;
5898                            return Ok(());
5899                        }
5900                    };
5901                    let executed = follow_up.execute().await;
5902                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5903                    if let Some(a) = adopted {
5904                        a.finish(&follow_up.state, executed);
5905                    }
5906                    if let Some(e) = failure {
5907                        self.state.event(
5908                            "fix",
5909                            format!(
5910                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5911                            ),
5912                        );
5913                    }
5914                    self.state.operator_fixes[request_index].follow_up_review_run =
5915                        Some(follow_up_id);
5916                }
5917                Err(e) => {
5918                    self.state.event(
5919                        "fix",
5920                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5921                    );
5922                }
5923            }
5924            self.state.save()?;
5925        }
5926
5927        Ok(())
5928    }
5929
5930    // --------------------------------------------------------------- review
5931
5932    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5933    /// names, in order, each at most once, moving on only when the call
5934    /// advances (an error, a quota hit or nothing usable - the same decision
5935    /// point `agent::chain_advances` is for the other chained roles).
5936    ///
5937    /// Each agent gets its own seat from `Runner::seat`: the same agent
5938    /// continues its conversation (the winner's own implementer seat when it
5939    /// is the winner's author, now that the competition is over), another
5940    /// takes a fresh one so the full prompt is sent again. A handover is
5941    /// recorded under `node`, which is what makes the fallback stick for the
5942    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5943    /// is returned, so an exhausted chain reads exactly like a single failed
5944    /// fixer: one quota loss, the same wording, the same refund. An earlier
5945    /// attempt's edits are left in the tree and judged, with the final
5946    /// attempt's, by what git says afterwards.
5947    ///
5948    /// Returns the job that produced the outcome, for `continue_fix_report`.
5949    async fn ask_fixer(
5950        &mut self,
5951        winner: &Candidate,
5952        node: &'static str,
5953        round: Option<usize>,
5954        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5955    ) -> (SeatJob, SeatState, AgentOutcome) {
5956        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5957        self.ask_fixer_among(attempts, node, round, None, build)
5958            .await
5959    }
5960
5961    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5962    /// worktree and a commit: before the next agent is asked after one that
5963    /// did not satisfy the call, the worktree is put back on that commit, so
5964    /// a failed agent's partial edits are not carried into the next one.
5965    async fn ask_fixer_among(
5966        &mut self,
5967        attempts: Vec<(AgentSpec, String)>,
5968        node: &'static str,
5969        round: Option<usize>,
5970        restore: Option<(&Path, &str)>,
5971        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5972    ) -> (SeatJob, SeatState, AgentOutcome) {
5973        let run_id = self.state.id.clone();
5974        let prompts = self.state.config.prompts.clone();
5975        let cache = self.state.config.cache_dir();
5976        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5977        let mut last = None;
5978        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5979            let seat = self.seat(&key, &spec.id);
5980            let mut job = build(spec.clone(), seat);
5981            if i > 0 {
5982                job.stem = format!("{}-{}", job.stem, spec.id);
5983            }
5984            let ctx = WaveCtx {
5985                carry_seats: false,
5986                pauses: Some((&self.pause, &self.interrupt)),
5987                run: &run_id,
5988                node,
5989                prompts: &prompts,
5990                cache: cache.as_deref(),
5991                round,
5992            };
5993            let (seat, out) =
5994                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5995            // A resource block never advances the chain: the chosen agent
5996            // was not asked, so it must be the one asked again on resume.
5997            let advances = match &out {
5998                AgentOutcome::Ok(o) => agent::output_advances(o),
5999                AgentOutcome::ResourceBlocked(_) => false,
6000                _ => true,
6001            };
6002            if let Some(next) = ids.get(i + 1)
6003                && advances
6004            {
6005                self.state.seats.insert(seat.key.clone(), seat.clone());
6006                let class =
6007                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6008                let reason = match &out {
6009                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6010                    other => fail_reason(other),
6011                };
6012                // A failed agent's edits must not reach the next one, and a
6013                // restore that did not work ends the chain: the call fails
6014                // and the caller's own (confirmed) restore decides.
6015                if let Some((wt, rev)) = restore
6016                    && let Err(e) = self.graft_restore(wt, rev).await
6017                {
6018                    let why =
6019                        format!("could not restore the worktree before the next fixer: {e:#}");
6020                    tracing::warn!("{why}");
6021                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6022                        AgentOutcome::ResourceBlocked(why)
6023                    } else {
6024                        AgentOutcome::Failed(why)
6025                    };
6026                    last = Some((job, seat, out));
6027                    break;
6028                }
6029                record_handover(
6030                    &mut self.state,
6031                    node,
6032                    &seat.key,
6033                    &spec.id,
6034                    next,
6035                    &class,
6036                    &reason,
6037                );
6038                continue;
6039            }
6040            last = Some((job, seat, out));
6041            break;
6042        }
6043        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6044        if let AgentOutcome::Quota(output) = &out
6045            && node != "graft"
6046        {
6047            // Graft is best effort: its caller records the loss and restores
6048            // the pre-graft tree, then proceeds to review and gate.
6049            // Every fallback was tried. Machine outages cannot be repaired by
6050            // spending a fix round; use each caller's resumable resource path.
6051            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6052            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6053            self.state.quota.push(loss);
6054            self.state.seats.insert(seat.key.clone(), seat.clone());
6055            self.state.status = RunStatus::Blocked;
6056            return (
6057                job,
6058                seat,
6059                AgentOutcome::ResourceBlocked(
6060                    "fixer unavailable; restore agent availability and resume".into(),
6061                ),
6062            );
6063        }
6064        (job, seat, out)
6065    }
6066
6067    /// The one repair a verification-only failure earns (see
6068    /// [`crate::recovery`]), before the first review round of a run that took
6069    /// a branch over.
6070    ///
6071    /// Saved as started *before* the fixer is called, so a resume never
6072    /// repairs twice; a start found on re-entry is rolled back to the commit
6073    /// it began from. A failed or unusable fixer leaves the tree as it was and
6074    /// the review goes on over it, so the first round's own verification
6075    /// reports the failure again. The failure is never carried into a round as
6076    /// a result: verification is re-run on whatever commit this leaves.
6077    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6078        use crate::recovery::{Mode, Repair};
6079        if !self.state.reviews.is_empty() {
6080            return Ok(());
6081        }
6082        let Some(mut rec) = self.state.recovery.clone() else {
6083            return Ok(());
6084        };
6085        if rec.mode != Mode::VerifyRepair {
6086            return Ok(());
6087        }
6088        let wt = winner.worktree.clone();
6089        let mut head = git::rev_parse(&wt, "HEAD").await?;
6090        match rec.repair.clone() {
6091            Repair::Done { .. } => return Ok(()),
6092            Repair::WaitingForRestore { before } => {
6093                self.graft_restore(&wt, &before).await?;
6094                head = before;
6095                rec.repair = Repair::NotStarted;
6096                self.state.recovery = Some(rec.clone());
6097                self.state.save()?;
6098            }
6099            Repair::Started { before } => {
6100                // Interrupted: the unfinished edits are not trusted.
6101                // An unconfirmed rollback blocks the run with `Started` kept,
6102                // so a resume retries it; the tree is never reviewed as is.
6103                if let Err(e) = self.graft_restore(&wt, &before).await {
6104                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6105                        return Err(e);
6106                    }
6107                    return self
6108                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6109                        .await;
6110                }
6111                rec.repair = Repair::Done {
6112                    before: before.clone(),
6113                    after: before,
6114                    committed: false,
6115                    failed: Some("interrupted; rolled back".to_owned()),
6116                };
6117                self.state
6118                    .event("recovery", "the repair was interrupted and rolled back");
6119                self.state.recovery = Some(rec);
6120                self.state.save()?;
6121                return Ok(());
6122            }
6123            Repair::NotStarted => {}
6124        }
6125        if crate::recovery::recheck(&mut rec, &head) {
6126            self.state.event("recovery", rec.summary());
6127            self.state.recovery = Some(rec);
6128            self.state.save()?;
6129            return Ok(());
6130        }
6131        rec.repair = Repair::Started {
6132            before: head.clone(),
6133        };
6134        self.state.recovery = Some(rec.clone());
6135        self.state.save()?;
6136
6137        let language = self.state.config.graph.language.clone();
6138        let sessions = self.state.config.graph.sessions;
6139        let artifacts = agent::artifacts_dir(&self.state.dir());
6140        let fix_prompt = prompt::recovery_fix(
6141            &self.state.instruction,
6142            &rec,
6143            self.state.config.graph.review_rounds,
6144            &language,
6145        );
6146        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6147        self.state.event(
6148            "recovery",
6149            format!(
6150                "repair: the fixer starts from the failed verification on {}",
6151                short(&head)
6152            ),
6153        );
6154        let (_job, seat, out) = self
6155            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6156                prompt: fix_prompt.clone(),
6157                spec,
6158                seat,
6159                cwd: wt.clone(),
6160                timeout,
6161                allow_write: true,
6162                sessions,
6163                artifacts: artifacts.clone(),
6164                stem: "fix-recovery".to_owned(),
6165                handover: None,
6166            })
6167            .await;
6168        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6169            rec.repair = Repair::WaitingForRestore { before: head };
6170            self.state.recovery = Some(rec);
6171            return self.stop_for_resource();
6172        }
6173        let failed = match &out {
6174            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6175            AgentOutcome::Quota(o) => {
6176                self.state.quota.push(QuotaLoss::from_report(
6177                    seat.key.clone(),
6178                    "fix",
6179                    o.quota.as_ref(),
6180                ));
6181                Some("rate limited (quota); the fixer could not run".to_owned())
6182            }
6183            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6184            AgentOutcome::Failed(e) => Some(e.clone()),
6185        };
6186        self.state.seats.insert(seat.key.clone(), seat);
6187        let coordination = self.coordinate(&wt).await?;
6188        let failed = match failed {
6189            // The agent's partial edits are not kept.
6190            Some(why) => {
6191                if let Err(e) = git::restore_to(&wt, &head).await {
6192                    return self
6193                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6194                        .await;
6195                }
6196                Some(why)
6197            }
6198            None => {
6199                if let Ok(r) = git::rescue_commit(
6200                    &wt,
6201                    "magi: recovery repair of the failed verification (uncommitted work)",
6202                )
6203                .await
6204                {
6205                    self.state.note_withheld("fix", &r.withheld);
6206                }
6207                None
6208            }
6209        };
6210        let after = git::rev_parse(&wt, "HEAD").await?;
6211        drop(coordination);
6212        self.state.event(
6213            "recovery",
6214            match &failed {
6215                Some(why) => format!("repair: nothing kept ({why})"),
6216                None if after != head => {
6217                    format!("repair: committed {}", short(&after))
6218                }
6219                None => "repair: the fixer made no commit".to_owned(),
6220            },
6221        );
6222        rec.repair = Repair::Done {
6223            before: head.clone(),
6224            committed: after != head,
6225            after,
6226            failed,
6227        };
6228        self.state.recovery = Some(rec);
6229        self.state.save()?;
6230        Ok(())
6231    }
6232
6233    /// A repair whose rollback could not be confirmed: the run is blocked and
6234    /// resumable, and the unconfirmed state is never reviewed.
6235    async fn stop_recovery(
6236        &mut self,
6237        mut rec: crate::recovery::RecoveryRecord,
6238        head: &str,
6239        why: String,
6240    ) -> Result<()> {
6241        // `Started` stays recorded: a resume retries the restore.
6242        rec.repair = crate::recovery::Repair::Started {
6243            before: head.to_owned(),
6244        };
6245        self.state.recovery = Some(rec);
6246        self.state
6247            .event("recovery", format!("repair could not be undone: {why}"));
6248        self.state.status = RunStatus::Blocked;
6249        self.state.save()?;
6250        bail!("recovery repair could not be rolled back: {why}")
6251    }
6252
6253    async fn review_loop(&mut self) -> Result<()> {
6254        if self.hold_base_sync().await? {
6255            return Ok(());
6256        }
6257        loop {
6258            if self
6259                .state
6260                .base_sync
6261                .as_ref()
6262                .is_some_and(|s| s.conflict.is_some())
6263            {
6264                return Ok(());
6265            }
6266            if self.catchup_repairable() {
6267                let record = self.state.reviews.last().expect("repairable round").clone();
6268                let winner = self
6269                    .state
6270                    .winner()
6271                    .cloned()
6272                    .context("catch-up repair has no winner")?;
6273                if !self.fix_review_round(&winner, record).await? {
6274                    return Ok(());
6275                }
6276            }
6277            self.review_loop_pass().await?;
6278            if !self.catchup_repairable() {
6279                return Ok(());
6280            }
6281        }
6282    }
6283
6284    /// Derive an owed catch-up repair from durable round evidence, including
6285    /// reentry after a crash before the fixer was dispatched.
6286    fn catchup_repairable(&self) -> bool {
6287        self.state.reviews.len() < self.state.config.graph.review_rounds
6288            && self.state.reviews.last().is_some_and(|r| {
6289                !r.clean
6290                    && r.blocking == 0
6291                    && r.fix.is_none()
6292                    && r.e2e_status() == E2eStatus::Failed
6293            })
6294    }
6295
6296    async fn review_loop_pass(&mut self) -> Result<()> {
6297        // A base that would not rebase is a person's decision, not a review
6298        // round: nothing here would change the answer, and reviewers and a
6299        // fixer would be spending real budget on a tree that cannot land
6300        // regardless of what they find.
6301        if self
6302            .state
6303            .base_sync
6304            .as_ref()
6305            .is_some_and(|s| s.conflict.is_some())
6306        {
6307            return Ok(());
6308        }
6309        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
6310        // agent files with `magi task add` name the run that paid for it. The
6311        // prompt overlay is cloned alongside it because the waves borrow it
6312        // while `self` is mutably borrowed by the node's own bookkeeping.
6313        let run_id = self.state.id.clone();
6314        let prompts = self.state.config.prompts.clone();
6315        let Some(winner) = self.state.winner().cloned() else {
6316            return Ok(());
6317        };
6318        let max_rounds = self.state.config.graph.review_rounds;
6319        // A clean round, an exhausted round budget, or a stalled tree (see
6320        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
6321        // they are recorded — recomputed here, not read off `status`, so a
6322        // reentry into a run that already stopped restates the identical
6323        // verdict instead of silently handing back whatever an earlier node
6324        // in this same walk clobbered `status` to (a solo-candidate
6325        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
6326        // below runs an empty range once the budget is spent, and would
6327        // otherwise fall through without touching `status` at all.
6328        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
6329            // A reentry after a crash between the last round's save and
6330            // `stop_reviewing` reaches the hand-off here, not there.
6331            if status == RunStatus::Gating {
6332                self.record_contested_handoff();
6333            }
6334            self.state.status = status;
6335            self.state.save()?;
6336            return Ok(());
6337        }
6338        self.state.status = RunStatus::Reviewing;
6339        // A last recorded round whose own verification never resolved
6340        // (`ResourceBlocked` — the shared build cache, not the patch) is
6341        // never a concluded round, whatever the round budget says: starting
6342        // a fresh round on top of it would spend a whole new reviewer wave
6343        // re-reading an unchanged patch instead of just retrying the one
6344        // check that actually needs it, and once the budget is spent the
6345        // loop below has nothing left to do at all (its range is empty).
6346        // Retry that check directly instead, exactly the same retry
6347        // `stop_reviewing` already does for its own catch-up case.
6348        if self.state.reviews.last().is_some_and(|r| {
6349            r.e2e_status() == E2eStatus::ResourceBlocked
6350                || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6351        }) {
6352            let shell = self.state.config.shell();
6353            return self
6354                .stop_unresolved_review(
6355                    "the last round's own verification never resolved",
6356                    &shell,
6357                    &winner.worktree,
6358                )
6359                .await;
6360        }
6361
6362        self.recovery_repair(&winner).await?;
6363        if self.resource_halt()? {
6364            return Ok(());
6365        }
6366
6367        let repo = self.state.repo.clone();
6368        let root = self.state.worktree_root();
6369        let language = self.state.config.graph.language.clone();
6370        let sessions = self.state.config.graph.sessions;
6371        let artifacts = self.merge_repair_artifacts();
6372        let base = self.landing_base();
6373        let base_short = short(&base);
6374        let reviewers = self.roles.reviewers.clone();
6375        let shell = self.state.config.shell();
6376
6377        for round in (self.state.reviews.len() + 1)..=max_rounds {
6378            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6379            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6380            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6381            // The prior round's own record, already persisted — never a
6382            // hand-carried variable of just its failing output: that is
6383            // exactly what let a round's e2e result drift out of sync with
6384            // which commit it was actually about (see `SCHEMA`'s doc for
6385            // schema 8). Judged against `head`, the commit reviewers are
6386            // about to look at now, so the summary always reads as "an
6387            // earlier head" here — this round's own patch has not been
6388            // checked yet.
6389            let prev_verification = self
6390                .state
6391                .reviews
6392                .last()
6393                .and_then(|r| r.verification_summary(&head));
6394            // Only the first round of a run that took a branch over: later
6395            // rounds have this run's own history.
6396            let prior_section = match (&self.state.recovery, round) {
6397                (Some(rec), 1) if self.state.reviews.is_empty() => {
6398                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6399                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6400                            Some(crate::recovery::head_of(
6401                                &git::diff(&winner.worktree, reviewed, "HEAD")
6402                                    .await
6403                                    .unwrap_or_default(),
6404                                crate::recovery::SINCE_DIFF_MAX,
6405                            ))
6406                        }
6407                        _ => None,
6408                    };
6409                    Some(prompt::recovery_review(rec, since.as_deref()))
6410                }
6411                _ => None,
6412            };
6413
6414            // Each reviewer gets its own detached checkout of exactly this
6415            // commit: nobody can perturb the winner's tree, and the fixer can
6416            // keep working without racing a reviewer.
6417            let mut jobs = Vec::new();
6418            let coordination = self.coordinate(&repo).await?;
6419            for (r, spec) in reviewers.iter().cloned().enumerate() {
6420                let wt = root.join(format!("review-{}", r + 1));
6421                // `clean -fdx` in a reset wipes setup's products, so every
6422                // round sets the seat up again.
6423                if wt.exists() {
6424                    git::reset_detached(&wt, &head).await?;
6425                } else {
6426                    git::worktree_add_detached(&repo, &wt, &head).await?;
6427                }
6428                self.setup_seat_worktree(&repo, &wt, false).await?;
6429                let seat_key = format!("review-{}", r + 1);
6430                // The seat starts the round on whoever answered it last, not
6431                // on the agent the spec names, so a failure is not re-paid.
6432                let spec = pick_start_spec(
6433                    &self.roles.reviewer_roster,
6434                    spec,
6435                    self.state.seat_history.get(&seat_key),
6436                );
6437                let seat = self.seat(&seat_key, &spec.id);
6438                jobs.push(SeatJob {
6439                    prompt: prompt::review(&prompt::ReviewCtx {
6440                        instruction: &self.state.instruction,
6441                        branch: &winner.branch,
6442                        base_short: &base_short,
6443                        stat: &stat,
6444                        patch: &patch,
6445                        verification: prev_verification.as_ref(),
6446                        prior: prior_section.as_deref(),
6447                        reviewers: reviewers.len(),
6448                        round,
6449                        rounds: max_rounds,
6450                        // A review-only run has no rankings, so nothing
6451                        // competed for this patch and the reviewer is told so.
6452                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6453                        lens: Lens::for_seat(r),
6454                        language: &language,
6455                        github_language: &self.state.config.graph.github_language,
6456                    }),
6457                    spec,
6458                    seat,
6459                    cwd: wt,
6460                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6461                    allow_write: false,
6462                    sessions,
6463                    artifacts: artifacts.clone(),
6464                    stem: format!("review-{round}-{}", r + 1),
6465                    handover: None,
6466                });
6467            }
6468
6469            drop(coordination);
6470            self.state.event(
6471                "review",
6472                format!(
6473                    "round {round}: {} reviewers on {}",
6474                    jobs.len(),
6475                    short(&head)
6476                ),
6477            );
6478            let mut quota_losses = Vec::new();
6479            let review_retries = self.state.config.graph.retries;
6480            let review_cache = self.state.config.cache_dir();
6481            let ctx = WaveCtx {
6482                carry_seats: true,
6483                pauses: Some((&self.pause, &self.interrupt)),
6484                run: &run_id,
6485                node: "review",
6486                prompts: &prompts,
6487                cache: review_cache.as_deref(),
6488                round: Some(round),
6489            };
6490            let results = ask_json_wave::<Review>(
6491                jobs,
6492                Arc::clone(&self.sem),
6493                review_retries,
6494                &self.roles.reviewer_roster,
6495                &ctx,
6496                &mut quota_losses,
6497                &mut self.state,
6498                &|_: &Review| Ok(()),
6499            )
6500            .await;
6501            if self.state.resource_block.is_some() {
6502                return self.stop_for_resource();
6503            }
6504            // Counted before the move below: how many of *this* round's
6505            // reviewer seats were lost to their own rate limit, as opposed to
6506            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6507            let round_quota_missing = quota_losses.len();
6508            self.state.quota.extend(quota_losses);
6509
6510            let mut records = Vec::new();
6511            let mut all_findings = Vec::new();
6512            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6513                let agent_id = seat.agent.clone();
6514                self.state.seats.insert(seat.key.clone(), seat);
6515                let mut record = ReviewRecord {
6516                    reviewer: r + 1,
6517                    agent: agent_id,
6518                    summary: String::new(),
6519                    findings: Vec::new(),
6520                    vote: None,
6521                    failed: None,
6522                    duration_ms: 0,
6523                    // Set for both outcomes: `failed: Some(_)` with
6524                    // `attempts > 0` is a seat every retry still lost, not a
6525                    // recovered one — only `failed: None` with `attempts > 0`
6526                    // reads as "answered after a nudge" (see this field's own
6527                    // doc).
6528                    attempts,
6529                };
6530                match res {
6531                    Ok((review, out)) => {
6532                        // Sanitized here, at the point every other piece of
6533                        // agent prose in this file is (candidate summaries,
6534                        // deliberation turns, vote reasons): a reviewer's own
6535                        // words are the one thing about it that could name
6536                        // it, and reconsideration below broadcasts this same
6537                        // summary and these same findings to every other
6538                        // seat on the panel.
6539                        record.summary =
6540                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6541                        record.vote = Some(review.vote);
6542                        record.duration_ms = out.duration_ms;
6543                        for (n, mut f) in review.findings.into_iter().enumerate() {
6544                            // ids are magi's, never the agent's: the fixer's
6545                            // adoption report is keyed by them.
6546                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6547                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6548                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6549                            // `file` is agent-supplied prose too, never
6550                            // checked against the real tree — the same
6551                            // exposure `title`/`detail` above have, just in
6552                            // a field easy to forget because it looks like a
6553                            // path rather than free text.
6554                            f.file = f
6555                                .file
6556                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6557                            all_findings.push(f.clone());
6558                            record.findings.push(f);
6559                        }
6560                        self.state.event(
6561                            "review",
6562                            format!(
6563                                "round {round}: reviewer {} voted {} with {} finding(s)",
6564                                r + 1,
6565                                review.vote.label(),
6566                                record.findings.len()
6567                            ),
6568                        );
6569                    }
6570                    Err(e) => {
6571                        record.failed = Some(e.to_string());
6572                        self.state.event(
6573                            "review",
6574                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6575                        );
6576                    }
6577                }
6578                records.push(record);
6579            }
6580
6581            // Tally the round's votes and, if they split, spend the one
6582            // round of reconsideration the split -> deliberate -> revote
6583            // shape `judge`/`vote` use for the panel, sized down to what a
6584            // read-only review round can afford: one round, and a revote
6585            // rather than an argument, because the panel already wrote its
6586            // reasoning down as findings the first time around.
6587            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6588            let vote_split =
6589                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6590            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6591            if vote_split {
6592                self.state.event(
6593                    "review",
6594                    format!(
6595                        "round {round}: votes split ({}) — one round of reconsideration",
6596                        initial_votes
6597                            .iter()
6598                            .map(|v| v.label())
6599                            .collect::<Vec<_>>()
6600                            .join(", ")
6601                    ),
6602                );
6603                // Seats read every seat's findings and votes, still numbered
6604                // and never named — the same anonymity `review` itself keeps.
6605                let panel: Vec<ReviewSeatReport<'_>> = records
6606                    .iter()
6607                    .filter_map(|r| {
6608                        r.vote.map(|vote| ReviewSeatReport {
6609                            reviewer: r.reviewer,
6610                            vote,
6611                            summary: &r.summary,
6612                            findings: &r.findings,
6613                        })
6614                    })
6615                    .collect();
6616
6617                let mut jobs = Vec::new();
6618                let mut seats_at = Vec::new();
6619                for (r, spec) in reviewers.iter().cloned().enumerate() {
6620                    // A seat with no initial vote has nothing to reconsider
6621                    // from and stays absent, the same as it stayed absent
6622                    // from `panel` above.
6623                    if records[r].vote.is_none() {
6624                        continue;
6625                    }
6626                    let wt = root.join(format!("review-{}", r + 1));
6627                    let seat_key = format!("review-{}", r + 1);
6628                    let spec = self.occupant(&seat_key, spec);
6629                    let seat = self.seat(&seat_key, &spec.id);
6630                    // A seat with no live session has already forgotten the
6631                    // initial review's prompt — restate the patch it is
6632                    // voting on, the same as `deliberate`/`vote` do for a
6633                    // judge in the same position.
6634                    // The panel already carries this seat's own review and
6635                    // vote, so restating the patch makes the prompt whole for
6636                    // a seat handed to another agent.
6637                    let build = |with_patch: bool| {
6638                        prompt::review_reconsider(&ReviewReconsiderCtx {
6639                            instruction: &self.state.instruction,
6640                            reviewer: r + 1,
6641                            lens: Lens::for_seat(r),
6642                            panel: &panel,
6643                            patch: with_patch.then_some(ReviewPatch {
6644                                branch: &winner.branch,
6645                                base_short: &base_short,
6646                                stat: &stat,
6647                                patch: &patch,
6648                            }),
6649                            round,
6650                            rounds: max_rounds,
6651                            language: &language,
6652                        })
6653                    };
6654                    let full = build(true);
6655                    let prompt = if has_context(&spec, &seat, sessions) {
6656                        build(false)
6657                    } else {
6658                        full.clone()
6659                    };
6660                    jobs.push(SeatJob {
6661                        prompt,
6662                        spec,
6663                        seat,
6664                        cwd: wt,
6665                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6666                        allow_write: false,
6667                        sessions,
6668                        artifacts: artifacts.clone(),
6669                        stem: format!("review-{round}-reconsider-{}", r + 1),
6670                        handover: Some(full),
6671                    });
6672                    seats_at.push(r);
6673                }
6674
6675                let mut recon_quota_losses = Vec::new();
6676                let recon_cache = self.state.config.cache_dir();
6677                let recon_ctx = WaveCtx {
6678                    carry_seats: true,
6679                    pauses: Some((&self.pause, &self.interrupt)),
6680                    run: &run_id,
6681                    node: "review",
6682                    prompts: &prompts,
6683                    cache: recon_cache.as_deref(),
6684                    round: Some(round),
6685                };
6686                let recon_results = ask_json_wave::<ReviewRevote>(
6687                    jobs,
6688                    Arc::clone(&self.sem),
6689                    review_retries,
6690                    &self.roles.reviewer_roster,
6691                    &recon_ctx,
6692                    &mut recon_quota_losses,
6693                    &mut self.state,
6694                    &|_: &ReviewRevote| Ok(()),
6695                )
6696                .await;
6697                if self.state.resource_block.is_some() {
6698                    return self.stop_for_resource();
6699                }
6700                self.state.quota.extend(recon_quota_losses);
6701
6702                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6703                    let agent_id = seat.agent.clone();
6704                    self.state.seats.insert(seat.key.clone(), seat);
6705                    let mut rec = ReviewRevoteRecord {
6706                        reviewer: r + 1,
6707                        agent: agent_id,
6708                        vote: None,
6709                        reason: String::new(),
6710                        failed: None,
6711                    };
6712                    match res {
6713                        Ok((rv, _)) => {
6714                            rec.vote = Some(rv.vote);
6715                            rec.reason =
6716                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6717                            self.state.event(
6718                                "review",
6719                                format!(
6720                                    "round {round}: reviewer {} revoted {}",
6721                                    r + 1,
6722                                    rv.vote.label()
6723                                ),
6724                            );
6725                        }
6726                        Err(e) => {
6727                            rec.failed = Some(e.to_string());
6728                            self.state.event(
6729                                "review",
6730                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6731                            );
6732                        }
6733                    }
6734                    reconsideration.push(rec);
6735                }
6736            } else if initial_votes.len() > 1 {
6737                self.state.event(
6738                    "review",
6739                    format!(
6740                        "round {round}: votes agreed ({}) — no reconsideration",
6741                        initial_votes[0].label()
6742                    ),
6743                );
6744            }
6745
6746            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6747            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6748            // A round that already has a blocking finding and a round left to
6749            // try is going back to the fixer no matter what `verify.e2e`
6750            // says, so running it first only spends the loop's slowest step
6751            // (minutes, for a Rust repo's full test suite) on a head about
6752            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6753            // runs once a round has no blocking findings left (see
6754            // `round_is_clean`, which a deferred — empty — `e2e` can never
6755            // satisfy since `blocking` is nonzero whenever this branch is
6756            // taken), and `stop_reviewing` forces a real run before it will
6757            // ever read a deferred round as green.
6758            let defer_e2e =
6759                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6760            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6761                let reason =
6762                    format!("{blocking} blocking finding(s) already required a fix this round");
6763                self.state.event(
6764                    "verify",
6765                    format!(
6766                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6767                         {}); it will run once a round has none left",
6768                        short(&head)
6769                    ),
6770                );
6771                (Vec::new(), false, true, Some(reason))
6772            } else {
6773                let e2e_commands = self.state.config.verify.e2e.clone();
6774                let cache_dir = self.state.config.cache_dir();
6775                let context = format!("round {round}");
6776                let wait = self.repository_wait();
6777                let (e2e, verify_retried) = with_cache_lease(
6778                    &mut self.state,
6779                    cache_dir.as_deref(),
6780                    "e2e",
6781                    "e2e",
6782                    &winner.worktree,
6783                    &head,
6784                    verify_timeout,
6785                    &context,
6786                    &wait,
6787                    |state, budget| {
6788                        let shell = shell.clone();
6789                        let e2e_commands = e2e_commands.clone();
6790                        let worktree = winner.worktree.clone();
6791                        let context = context.clone();
6792                        async move {
6793                            run_e2e_with_retry(
6794                                state,
6795                                &shell,
6796                                &e2e_commands,
6797                                &worktree,
6798                                budget,
6799                                &context,
6800                            )
6801                            .await
6802                        }
6803                    },
6804                )
6805                .await?;
6806                (e2e, verify_retried, false, None)
6807            };
6808
6809            let expected = records.len();
6810            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6811            let incomplete = answered < expected;
6812            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6813            let policy = self.state.config.graph.incomplete_review;
6814            let clean = round_is_clean(
6815                blocking,
6816                e2e_ok,
6817                answered,
6818                expected,
6819                round_quota_missing,
6820                policy,
6821            );
6822
6823            let mut round_record = ReviewRound {
6824                round,
6825                head: head.clone(),
6826                verified_head: None,
6827                verified_at: None,
6828                reviews: records,
6829                e2e,
6830                verify_retried,
6831                e2e_deferred,
6832                e2e_defer_reason,
6833                fix: None,
6834                blocking,
6835                answered,
6836                expected,
6837                clean,
6838                progressed: false,
6839                vote_split,
6840                reconsideration,
6841                verdict: None,
6842            };
6843            // The final vote per seat is its revote where reconsideration
6844            // ran and answered, its initial vote otherwise — the same
6845            // fallback `tally` uses for a judge whose private vote failed.
6846            round_record.verdict = ReviewVote::worst(
6847                round_record
6848                    .final_votes()
6849                    .into_iter()
6850                    .map(|(_, _, vote)| vote),
6851            );
6852            // Which commit and when magi actually attempted to check —
6853            // known the moment a command was dispatched against `head`,
6854            // whether or not it finished: a resource-blocked attempt still
6855            // targeted a specific commit at a specific time, and leaving
6856            // that unrecorded is exactly what made `verification_summary`
6857            // report a fresh attempt as "commit unknown ... recorded before
6858            // this was tracked", indistinguishable from a genuinely old,
6859            // untracked record. Only a deferred or unconfigured round never
6860            // ran at all and has nothing to record — see
6861            // `ReviewRound::verified_head`'s own doc.
6862            if !matches!(
6863                round_record.e2e_status(),
6864                E2eStatus::Deferred | E2eStatus::NotConfigured
6865            ) {
6866                round_record.verified_head = Some(head.clone());
6867                round_record.verified_at = Some(Timestamp::now());
6868            }
6869
6870            if incomplete {
6871                let missing: Vec<String> = round_record
6872                    .reviews
6873                    .iter()
6874                    .filter(|r| r.failed.is_some())
6875                    .map(|r| format!("review-{}", r.reviewer))
6876                    .collect();
6877                self.state.event(
6878                    "review",
6879                    format!(
6880                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6881                        missing.join(", ")
6882                    ),
6883                );
6884            }
6885
6886            if clean {
6887                self.state.event(
6888                    "review",
6889                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6890                        format!(
6891                            "round {round}: clean (warn policy, incomplete panel) — no \
6892                             blocking findings from the seats that answered, verification green"
6893                        )
6894                    } else if incomplete {
6895                        format!(
6896                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6897                             quorum) — no blocking findings from the seats that answered, \
6898                             verification green",
6899                            expected - answered
6900                        )
6901                    } else {
6902                        format!("round {round}: clean — no blocking findings, verification green")
6903                    },
6904                );
6905                self.state.reviews.push(round_record);
6906                self.state.status = RunStatus::Gating;
6907                self.state.save()?;
6908                return Ok(());
6909            }
6910
6911            // Nothing was raised and verification passed, but not every seat
6912            // answered and `round_is_clean` still refused to call it clean —
6913            // either a seat is missing for a reason other than its own quota
6914            // (a crash, a timeout, unparsable output — worth another try), or
6915            // every seat that could have answered lost its quota and nobody
6916            // is left to decide on: re-review rather than send the fixer
6917            // after a round with nothing to fix.
6918            if incomplete && blocking == 0 && e2e_ok {
6919                self.state.reviews.push(round_record);
6920                self.state.save()?;
6921                if round == max_rounds {
6922                    self.state.status = RunStatus::Blocked;
6923                    self.state.event(
6924                        "review",
6925                        format!(
6926                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6927                             refusing to call it clean",
6928                            expected - answered
6929                        ),
6930                    );
6931                    return Ok(());
6932                }
6933                continue;
6934            }
6935
6936            // Nothing for the fixer to act on (`blocking == 0`) and the only
6937            // reason this round is not clean is that magi itself never got
6938            // a command to run — the shared build cache, not the patch (see
6939            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6940            // the fixer would invite a change to appease contention that has
6941            // nothing to do with the diff, and would leave this attempt
6942            // sitting in the next round's prompt as if it were about an
6943            // earlier, superseded commit rather than what it actually is:
6944            // the same head, still waiting to be checked. Wait for it the
6945            // same way the final round's own contention is already handled,
6946            // whatever round this happens to be.
6947            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6948                self.state.reviews.push(round_record);
6949                return self
6950                    .stop_unresolved_review(
6951                        "the round's own verification could not run",
6952                        &shell,
6953                        &winner.worktree,
6954                    )
6955                    .await;
6956            }
6957
6958            if round == max_rounds {
6959                self.state.reviews.push(round_record);
6960                return self
6961                    .stop_reviewing(
6962                        &format!(
6963                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6964                        ),
6965                        &shell,
6966                        &winner.worktree,
6967                    )
6968                    .await;
6969            }
6970
6971            if !self.fix_review_round(&winner, round_record).await? {
6972                return Ok(());
6973            }
6974        }
6975        Ok(())
6976    }
6977
6978    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
6979    /// False means admission or continuation policy stopped the review loop.
6980    async fn fix_review_round(
6981        &mut self,
6982        winner: &Candidate,
6983        mut round_record: ReviewRound,
6984    ) -> Result<bool> {
6985        let round = round_record.round;
6986        let shell = self.state.config.shell();
6987        let step = self.run_fix_round(winner, &mut round_record).await?;
6988        if matches!(step, FixStep::Resource) {
6989            self.stop_for_resource()?;
6990            return Ok(false);
6991        }
6992        if self.state.reviews.len() < round {
6993            self.state.reviews.push(round_record);
6994        } else {
6995            self.state.reviews[round - 1] = round_record;
6996        }
6997        self.state.save()?;
6998
6999        // The fixer's own report never came back this round, even after
7000        // `continue_fix_report`'s own budget was spent on it — not an
7001        // ordinary "no report" (dropped stream, quota, plain failure),
7002        // which already reads that way and is left to the existing round
7003        // budget. Stopping here, rather than opening another round, is
7004        // what keeps a next reviewer/fixer wave from ever being
7005        // dispatched onto `winner.worktree` while whatever the seat's
7006        // last call may still have running there is unaccounted for: no
7007        // process liveness check exists (and none is being added — see
7008        // AGENTS.md/this task's own scope), so the only way to honour
7009        // "nothing starts before a valid report returns" is to not start
7010        // anything further on this worktree from this run at all.
7011        if matches!(step, FixStep::Recorded { report_lost: true }) {
7012            self.stop_reviewing(
7013                "the fixer's adoption report never came back, even after resuming its \
7014                         own seat; refusing to start another round against the same worktree \
7015                         while that is unresolved",
7016                &shell,
7017                &winner.worktree,
7018            )
7019            .await?;
7020            return Ok(false);
7021        }
7022
7023        let streak = self
7024            .state
7025            .reviews
7026            .iter()
7027            .rev()
7028            .take_while(|r| !r.progressed)
7029            .count();
7030        if streak >= STAGNANT_LIMIT {
7031            self.stop_reviewing(
7032                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7033                &shell,
7034                &winner.worktree,
7035            )
7036            .await?;
7037            return Ok(false);
7038        }
7039        Ok(true)
7040    }
7041
7042    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7043    async fn run_fix_round(
7044        &mut self,
7045        winner: &Candidate,
7046        round_record: &mut ReviewRound,
7047    ) -> Result<FixStep> {
7048        let round = round_record.round;
7049        let max_rounds = self.state.config.graph.review_rounds;
7050        let run_id = self.state.id.clone();
7051        let prompts = self.state.config.prompts.clone();
7052        let language = self.state.config.graph.language.clone();
7053        let sessions = self.state.config.graph.sessions;
7054        let artifacts = self.merge_repair_artifacts();
7055        let base = self.landing_base();
7056        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7057        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7058        let this_round_verification = round_record.verification_summary(&head);
7059        let all_findings: Vec<_> = round_record
7060            .reviews
7061            .iter()
7062            .flat_map(|r| r.findings.iter().cloned())
7063            .collect();
7064        // Fix. The winner's own implementer seat continues its conversation:
7065        // the competition is over, so context is pure benefit now.
7066        let blocking_findings: Vec<_> = all_findings
7067            .iter()
7068            .filter(|f| f.severity.blocks())
7069            .cloned()
7070            .collect();
7071        let fix_prompt = prompt::fix_in(
7072            &self.state.instruction,
7073            &blocking_findings,
7074            this_round_verification.as_ref(),
7075            round,
7076            max_rounds,
7077            &language,
7078            &self.state.config.graph.github_language,
7079        );
7080        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7081        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7082        let (job, seat, out) = self
7083            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7084                prompt: fix_prompt.clone(),
7085                spec,
7086                seat,
7087                cwd: winner.worktree.clone(),
7088                timeout,
7089                allow_write: true,
7090                sessions,
7091                artifacts: artifacts.clone(),
7092                stem: format!("fix-{round}"),
7093                handover: None,
7094            })
7095            .await;
7096        // Not asked: the round is not recorded (so a resume repeats it),
7097        // no fixer was handed over to, and the seat is left as it was.
7098        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7099            return Ok(FixStep::Resource);
7100        }
7101        let agent_id = seat.agent.clone();
7102
7103        let mut fix = FixRecord {
7104            agent: agent_id,
7105            addressed: Vec::new(),
7106            rejected: Vec::new(),
7107            notes: String::new(),
7108            committed: false,
7109            failed: None,
7110            duration_ms: 0,
7111            continuation: None,
7112        };
7113        let mut continuation = ContinuationRecord::not_needed();
7114        let mut final_seat = seat.clone();
7115        match out {
7116            AgentOutcome::Ok(o) => {
7117                fix.duration_ms = o.duration_ms;
7118                let parsed = verdict::extract_json::<FixReport>(&o.text);
7119                // A parsed report standing next to a command this same
7120                // reply's own CLI never confirmed the exit status of is
7121                // not a resolved answer — the identical `CommandEvidence`
7122                // `state.jobs` renders, read here instead of only on
7123                // display, per the completion judgment and the shown
7124                // record needing to agree.
7125                let incomplete_reason = match &parsed {
7126                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7127                        "the reply parsed, but it reported a command whose own CLI never \
7128                             confirmed an exit status"
7129                            .to_owned(),
7130                    ),
7131                    Ok(_) => None,
7132                    Err(e) => Some(e.to_string()),
7133                };
7134                match incomplete_reason {
7135                    None => {
7136                        let report = parsed.expect("checked Ok above");
7137                        fix.addressed = report.addressed;
7138                        fix.rejected = report.rejected;
7139                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7140                    }
7141                    Some(reason) => {
7142                        let (resumed_seat, resolved, failure, cont) = self
7143                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7144                            .await;
7145                        fix.duration_ms += cont.cumulative_wait_ms;
7146                        continuation = cont;
7147                        final_seat = resumed_seat;
7148                        match resolved {
7149                            Some(report) => {
7150                                fix.addressed = report.addressed;
7151                                fix.rejected = report.rejected;
7152                                fix.notes =
7153                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
7154                            }
7155                            None => fix.failed = failure,
7156                        }
7157                    }
7158                }
7159            }
7160            // The CLI's raw error JSON is not a fix report to parse.
7161            AgentOutcome::Dropped(o) => {
7162                fix.duration_ms = o.duration_ms;
7163                let why = o
7164                    .dropped
7165                    .as_ref()
7166                    .map(|d| d.why.as_str())
7167                    .unwrap_or("the CLI ended the stream without delivering its answer");
7168                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7169            }
7170            AgentOutcome::Quota(o) => {
7171                self.state.quota.push(QuotaLoss::from_report(
7172                    final_seat.key.clone(),
7173                    "fix",
7174                    o.quota.as_ref(),
7175                ));
7176                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7177            }
7178            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7179        }
7180        if self.state.resource_block.is_some() {
7181            self.state.seats.insert(final_seat.key.clone(), final_seat);
7182            return Ok(FixStep::Resource);
7183        }
7184        fix.continuation = Some(continuation);
7185        self.state.seats.insert(final_seat.key.clone(), final_seat);
7186        let coordination = self.coordinate(&winner.worktree).await?;
7187        if let Ok(r) = git::rescue_commit(
7188            &winner.worktree,
7189            &format!("magi: review round {round} fixes (uncommitted work)"),
7190        )
7191        .await
7192        {
7193            self.state.note_withheld("fix", &r.withheld);
7194        }
7195        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7196        fix.committed = after != before;
7197        // Judged by what `git` says moved against base, never by the
7198        // fixer's own `addressed`/`rejected` count — see
7199        // `ReviewRound::progressed`. Propagated with `?`, the same as the
7200        // `patch` snapshot above: swallowing this error would default
7201        // `diff_after` to empty, which almost always differs from a
7202        // non-empty `patch` and reads as "progressed" — exactly backwards
7203        // for a `git` failure the stagnation check cannot see through.
7204        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7205        drop(coordination);
7206        let progressed = diff_after != patch;
7207        let commit_note = if fix.committed {
7208            "committed"
7209        } else {
7210            "NO new commit"
7211        };
7212        let tree_note = if progressed {
7213            "changed vs base"
7214        } else {
7215            "unchanged vs base"
7216        };
7217        self.state.event(
7218            "fix",
7219            match &fix.failed {
7220                // Distinct on purpose from "0 addressed, 0 rejected": the
7221                // fixer's own diff still landed (blocking counts do keep
7222                // falling round over round), only its adoption report did
7223                // not come back, so this must never read like every
7224                // finding was reviewed and declined.
7225                Some(reason) => {
7226                    format!(
7227                        "round {round}: fixer's adoption report was lost ({reason}); \
7228                             {commit_note}, tree {tree_note}"
7229                    )
7230                }
7231                None => format!(
7232                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7233                         {tree_note}{}",
7234                    fix.addressed.len(),
7235                    fix.rejected.len(),
7236                    if continuation.outcome == ContinuationOutcome::Resumed {
7237                        format!(
7238                            " (adoption report recovered after {} continuation(s))",
7239                            continuation.attempts
7240                        )
7241                    } else {
7242                        String::new()
7243                    },
7244                ),
7245            },
7246        );
7247        round_record.fix = Some(fix);
7248        round_record.progressed = progressed;
7249        Ok(FixStep::Recorded {
7250            report_lost: matches!(
7251                continuation.outcome,
7252                ContinuationOutcome::Exhausted
7253                    | ContinuationOutcome::QuotaLost
7254                    | ContinuationOutcome::NoSession
7255            ),
7256        })
7257    }
7258
7259    /// Decide, from the last recorded round's own verification, whether
7260    /// stopping the review loop is a hand-off or a genuine block.
7261    ///
7262    /// Called once the loop has given up trying — the round budget is spent,
7263    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
7264    /// findings still open, never while a round is still clean or the
7265    /// incomplete-panel case handled inline above. Gate and e2e are facts
7266    /// about the tree; a lingering review finding is an opinion, and this
7267    /// workload's own `magi stats` puts reviewer precision low enough
7268    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
7269    /// must not by itself stand between a green, verified change and the
7270    /// human who decides what to do with it. A red e2e is not an opinion, so
7271    /// that case still blocks, with the failing command and a tail of its
7272    /// output recorded here rather than left in `run.json` for someone to go
7273    /// find.
7274    ///
7275    /// A round that deferred its own e2e (see [`Config::graph`]'s
7276    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
7277    /// only because nothing ran, and treating an empty list as a passing one
7278    /// here is exactly the "deferred painted green" bug this function exists
7279    /// to not have. When the last round's own verification never resolved —
7280    /// deferred on purpose, or a real attempt the shared build cache blocked
7281    /// — this makes (or retries) the real run, on the actual worktree this
7282    /// loop is about to stop touching, before deciding anything. A
7283    /// resource-blocked attempt is likewise never read as either green or
7284    /// red: it is evidence about the machine, not the patch (see
7285    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
7286    /// blocked cache leaves this call without deciding rather than guessing
7287    /// — the caller retries on a later reentry.
7288    /// Record, once, that the review loop handed off over a blocking finding
7289    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
7290    /// `land` asks the owner even with `land_approval` off. Called from every
7291    /// path that concludes `Gating`; a reentry keeps the first record.
7292    fn record_contested_handoff(&mut self) {
7293        if self.state.contested_handoff.is_some() {
7294            return;
7295        }
7296        let Some(contested) = self
7297            .state
7298            .reviews
7299            .last()
7300            .and_then(ReviewRound::contested_handoff)
7301        else {
7302            return;
7303        };
7304        self.state.event(
7305            "review",
7306            format!(
7307                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7308                 wait for the owner's approval",
7309                contested.findings.len(),
7310                contested.rejecters.len()
7311            ),
7312        );
7313        self.state.contested_handoff = Some(contested);
7314    }
7315
7316    async fn stop_unresolved_review(
7317        &mut self,
7318        why: &str,
7319        shell: &[String],
7320        worktree: &Path,
7321    ) -> Result<()> {
7322        self.stop_reviewing_inner(why, shell, worktree, true).await
7323    }
7324
7325    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7326        self.stop_reviewing_inner(why, shell, worktree, false).await
7327    }
7328
7329    async fn stop_reviewing_inner(
7330        &mut self,
7331        why: &str,
7332        shell: &[String],
7333        worktree: &Path,
7334        repairable: bool,
7335    ) -> Result<()> {
7336        let round_idx = self.state.reviews.len() - 1;
7337        // A deferred round and a resource-blocked one are the same shape
7338        // here: neither has a real result yet, and both get one more
7339        // attempt. Read off `e2e_status` — the single source for this —
7340        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
7341        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
7342        // still retries instead of being read as a settled result the
7343        // instant it stops being empty.
7344        let needs_catchup_run = matches!(
7345            self.state.reviews[round_idx].e2e_status(),
7346            E2eStatus::Deferred | E2eStatus::ResourceBlocked
7347        );
7348        if needs_catchup_run {
7349            let round = self.state.reviews[round_idx].round;
7350            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7351            let commands = self.state.config.verify.e2e.clone();
7352            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7353            let cache_dir = self.state.config.cache_dir();
7354            let context = format!(
7355                "round {round}: verification unresolved, catching up before the final decision"
7356            );
7357            let wait = self.repository_wait();
7358            let (outcomes, verify_retried) = with_cache_lease(
7359                &mut self.state,
7360                cache_dir.as_deref(),
7361                "e2e",
7362                "e2e",
7363                worktree,
7364                &attempted_head,
7365                timeout,
7366                &context,
7367                &wait,
7368                |state, budget| {
7369                    let shell = shell.to_vec();
7370                    let commands = commands.clone();
7371                    let context = context.clone();
7372                    async move {
7373                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7374                            .await
7375                    }
7376                },
7377            )
7378            .await?;
7379            let last = &mut self.state.reviews[round_idx];
7380            last.e2e = outcomes;
7381            last.verify_retried = verify_retried;
7382            // Always the commit and time this attempt actually targeted,
7383            // whether or not it happens to equal the reviewed `head` and
7384            // whether or not a command finished — see
7385            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
7386            // attempt is recorded too, so a later reader sees "attempted
7387            // again at T2" rather than silence.
7388            last.verified_head = Some(attempted_head);
7389            last.verified_at = Some(Timestamp::now());
7390            if verify_inconclusive(&last.e2e) {
7391                // Still not a real result: `e2e_deferred` is left exactly
7392                // as it was, so `needs_catchup_run` above reads
7393                // `ResourceBlocked` (via `e2e_status`, which checks
7394                // `resource_blocked` before `e2e_deferred`) and retries
7395                // again on the next reentry, rather than recording
7396                // contention as a red e2e and blocking the run on it.
7397                self.state.save()?;
7398                return Ok(());
7399            }
7400            last.e2e_deferred = false;
7401        }
7402        let last = &self.state.reviews[round_idx];
7403        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7404
7405        match last.e2e_status() {
7406            E2eStatus::Failed => {
7407                let red: Vec<String> = last
7408                    .e2e
7409                    .iter()
7410                    .filter(|o| !o.ok())
7411                    .map(|o| {
7412                        format!(
7413                            "`{}` -> {:?}\n{}",
7414                            o.command,
7415                            o.code,
7416                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7417                        )
7418                    })
7419                    .collect();
7420                let can_repair = last.blocking == 0 && last.fix.is_none();
7421                self.state.reviews[round_idx].clean = false;
7422                self.state
7423                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7424                self.state.status = RunStatus::Blocked;
7425                if repairable
7426                    && can_repair
7427                    && self.state.reviews.len() < self.state.config.graph.review_rounds
7428                {
7429                    self.state.status = RunStatus::Reviewing;
7430                }
7431            }
7432            // `needs_catchup_run` above already retried once this call; if
7433            // it is still blocked, this is magi's own admission it could
7434            // not get a command to run, never a verdict on the patch — the
7435            // run is left exactly where a later reentry can retry again.
7436            E2eStatus::ResourceBlocked => {
7437                self.state.event(
7438                    "review",
7439                    format!(
7440                        "{why}; e2e could not run (resource unavailable); not \
7441                         deciding yet"
7442                    ),
7443                );
7444            }
7445            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7446                self.state.event(
7447                    "review",
7448                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7449                );
7450                self.record_contested_handoff();
7451                self.state.status = RunStatus::Gating;
7452            }
7453        }
7454        self.state.save()?;
7455        Ok(())
7456    }
7457
7458    // ----------------------------------------------------------------- gate
7459
7460    async fn gate(&mut self) -> Result<()> {
7461        if self.hold_base_sync().await? {
7462            return Ok(());
7463        }
7464        // Judged by the review record itself, not by `status`: a solo
7465        // candidate's `judge`/`deliberate` skip rewrites `status` on every
7466        // reentry (see `judge`), and trusting it here is exactly how a run
7467        // that exhausted its review budget got gated and merged a second
7468        // time around. `review_conclusion` recomputes the review loop's own
7469        // verdict from the round records themselves — `Gating` for a clean
7470        // round or a hand-off (see `stop_reviewing`), anything else means the
7471        // loop is still going or genuinely blocked.
7472        // A base the winner could not be replayed onto is a decision, not a
7473        // round: there is no landing tree to gate. Read as its own record for
7474        // the same reason the review verdict is.
7475        if self.state.status == RunStatus::Failed
7476            || self
7477                .state
7478                .base_sync
7479                .as_ref()
7480                .is_some_and(|s| s.conflict.is_some())
7481            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7482                != Some(RunStatus::Gating)
7483        {
7484            return Ok(());
7485        }
7486        let Some(winner) = self.state.winner().cloned() else {
7487            return Ok(());
7488        };
7489        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7490        self.state.reconcile_verification_head(&head);
7491        self.state.save()?;
7492        let wait = self.repository_wait();
7493        if !verify_current_head(&mut self.state, &winner, &wait).await? {
7494            return Ok(());
7495        }
7496        if self.state.gate_ran {
7497            // `review_loop` derives its conclusion from the clean review
7498            // record on every reentry and therefore puts a completed run back
7499            // in `Gating`. A recorded gate is a stronger, terminal fact:
7500            // retain its original command output (or lack of any, for a repo
7501            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
7502            // doc) and restore `Blocked` on a real failure rather than
7503            // pretending the command is still running or running it a second
7504            // time. `gate_ran == false` remains the only shape — unattempted,
7505            // or a resource-blocked retry — that may still need to execute a
7506            // command.
7507            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7508                self.state.status = RunStatus::Blocked;
7509                self.state.save()?;
7510            }
7511            return Ok(());
7512        }
7513        self.state.status = RunStatus::Gating;
7514        let mut outcomes = self.run_gate(&winner).await?;
7515        let mut attempted_head = self.state.gate_head.clone();
7516        loop {
7517            // A resource-blocked outcome means the gate command never actually
7518            // ran - the shared build cache could not be acquired or confirmed
7519            // fresh in time - which is evidence about the machine, not about
7520            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
7521            // Recording it as a red gate would mark a run `Blocked` on nothing
7522            // but contention magi has already logged; leaving `self.state.gate`
7523            // empty and `self.state.gate_ran` false instead keeps the shape
7524            // this function already treats as "still needs to run" (see the
7525            // early-return above), so the next call retries the command
7526            // rather than concluding anything.
7527            if verify_inconclusive(&outcomes) {
7528                self.state.save()?;
7529                return Ok(());
7530            }
7531            if outcomes.iter().all(CommandOutcome::ok) {
7532                break;
7533            }
7534            let fixes_before = self.state.gate_fixes.len();
7535            match self.gate_fix_round(&winner, &outcomes).await? {
7536                GateFix::Retry => {
7537                    outcomes = self.run_gate(&winner).await?;
7538                    attempted_head = self.state.gate_head.clone();
7539                }
7540                GateFix::Stop => {
7541                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
7542                    if self.state.gate_fixes.len() > fixes_before
7543                        && attempted_head.as_deref() != Some(current.as_str())
7544                        && verify_current_head(&mut self.state, &winner, &wait).await?
7545                    {
7546                        // A new SHA owes a gate even when a spent fixer changed only
7547                        // commit metadata. This does not grant another repair round.
7548                        outcomes = self.run_gate(&winner).await?;
7549                        attempted_head = self.state.gate_head.clone();
7550                    } else {
7551                        break;
7552                    }
7553                }
7554                GateFix::Defer => {
7555                    self.state.save()?;
7556                    return Ok(());
7557                }
7558            }
7559        }
7560        let passed = outcomes.iter().all(CommandOutcome::ok);
7561        self.state.gate = outcomes;
7562        self.state.gate_ran = true;
7563        self.state.gate_head = attempted_head;
7564        self.state.save()?;
7565        if !verify_current_head(&mut self.state, &winner, &wait).await? {
7566            return Ok(());
7567        }
7568        if !passed {
7569            self.state.status = RunStatus::Blocked;
7570            let spent = self.state.gate_fixes.len();
7571            self.state.event(
7572                "gate",
7573                if spent == 0 {
7574                    "gate failed; not merging".to_owned()
7575                } else {
7576                    format!("gate failed after {spent} gate-fix round(s); not merging")
7577                },
7578            );
7579        }
7580        self.state.save()?;
7581        Ok(())
7582    }
7583
7584    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7585    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7586    /// after review is clean and never on a candidate awaiting judging.
7587    ///
7588    /// Never fails the run: a non-zero exit or timeout is a warning and a
7589    /// recorded outcome, and the gate remains the single arbiter. Nothing
7590    /// configured means nothing happens - no event, no commit. `commit_all`
7591    /// commits any leftover change under the neutral identity and returns
7592    /// `false` when the tree is clean, so no empty commit is ever made.
7593    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7594        let commands = state.config.verify.pre_gate.clone();
7595        if commands.is_empty() {
7596            return;
7597        }
7598        let shell = state.config.shell();
7599        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7600        let (outcomes, _) = run_commands(
7601            state,
7602            "pre_gate",
7603            "pre_gate",
7604            0,
7605            &shell,
7606            &commands,
7607            &winner.worktree,
7608            timeout,
7609        )
7610        .await;
7611        for o in &outcomes {
7612            if !o.ok() {
7613                tracing::warn!(
7614                    "pre_gate `{}` failed ({:?}); the gate decides",
7615                    o.command,
7616                    o.code
7617                );
7618            }
7619            state.event(
7620                "pre_gate",
7621                format!(
7622                    "`{}` -> {}",
7623                    o.command,
7624                    if o.ok() {
7625                        "pass".to_owned()
7626                    } else {
7627                        format!(
7628                            "FAIL ({:?})\n{}",
7629                            o.code,
7630                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7631                        )
7632                    }
7633                ),
7634            );
7635        }
7636        state.pre_gate = outcomes;
7637        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7638            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7639                Ok(head) => {
7640                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7641                    state.pre_gate_commit = Some(head);
7642                }
7643                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7644            },
7645            Ok(false) => {}
7646            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7647        }
7648        if let Err(e) = state.save() {
7649            tracing::warn!("could not persist the pre_gate record: {e:#}");
7650        }
7651    }
7652
7653    /// Run `verify.gate` once against the winner's current tree, logging one
7654    /// event per command. Empty when nothing is configured.
7655    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7656        let wait = self.repository_wait();
7657        run_landing_gate(&mut self.state, winner, &wait).await
7658    }
7659
7660    /// One bounded fix round for a failing gate.
7661    ///
7662    /// The fixer is told the failure came from the gate itself, not from a
7663    /// reviewer, and is shown the failed commands, their exit codes and a tail
7664    /// of their output - whatever `[verify].gate` holds, nothing here knows
7665    /// what those commands run. Only a normal non-zero exit that printed
7666    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7667    /// command or a full disk says nothing about the code, and a fixer sent
7668    /// after it can only appease the machine. The round is judged by what git
7669    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7670    /// again before the gate does, so a fix cannot trade a green gate for a
7671    /// red e2e unnoticed.
7672    async fn gate_fix_round(
7673        &mut self,
7674        winner: &Candidate,
7675        outcomes: &[CommandOutcome],
7676    ) -> Result<GateFix> {
7677        let cap = self.state.config.graph.gate_fix_rounds;
7678        let spent = self.state.gate_fixes.len();
7679        if spent >= cap {
7680            if cap > 0 {
7681                self.state.event(
7682                    "gate",
7683                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7684                );
7685            }
7686            return Ok(GateFix::Stop);
7687        }
7688        if !gate_fixable(outcomes) {
7689            self.state.event(
7690                "gate",
7691                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7692                 command or similar); not spending a fix round on it",
7693            );
7694            return Ok(GateFix::Stop);
7695        }
7696        let min_free = self.state.config.disk.min_free_bytes;
7697        if min_free > 0 {
7698            match crate::disk::free_bytes(&winner.worktree) {
7699                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7700                Ok(free) => {
7701                    self.state.event(
7702                        "gate",
7703                        format!(
7704                            "only {free} bytes free ({min_free} required by `[disk] \
7705                             min_free_bytes`); not spending a fix round on a failure the disk \
7706                             may explain"
7707                        ),
7708                    );
7709                    return Ok(GateFix::Stop);
7710                }
7711                Err(e) => {
7712                    self.state.event(
7713                        "gate",
7714                        format!("free disk space could not be measured ({e:#}); no fix round"),
7715                    );
7716                    return Ok(GateFix::Stop);
7717                }
7718            }
7719        }
7720
7721        let attempt = spent + 1;
7722        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7723        let base = self.landing_base();
7724        let fix_prompt = prompt::gate_fix_in(
7725            &self.state.instruction,
7726            &failed,
7727            attempt,
7728            cap,
7729            &self.state.config.graph.language,
7730            &self.state.config.graph.github_language,
7731        );
7732        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7733        let sessions = self.state.config.graph.sessions;
7734        let artifacts = self.merge_repair_artifacts();
7735        self.state.event(
7736            "gate",
7737            format!("gate failed; gate-fix round {attempt} of {cap}"),
7738        );
7739        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7740        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7741        // Persist before invoking the fixer: it may commit and then be
7742        // interrupted before its result (or the following e2e) is saved.
7743        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7744            head: before.clone(),
7745            passed: false,
7746            outcomes: Vec::new(),
7747        });
7748        self.state.save()?;
7749        let (_, seat, out) = self
7750            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7751                prompt: fix_prompt.clone(),
7752                spec,
7753                seat,
7754                cwd: winner.worktree.clone(),
7755                timeout,
7756                allow_write: true,
7757                sessions,
7758                artifacts: artifacts.clone(),
7759                stem: format!("gate-fix-{attempt}"),
7760                handover: None,
7761            })
7762            .await;
7763        // The fixer was never asked: no round is spent (nothing is recorded)
7764        // and a later reentry asks the same agent again.
7765        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7766            return Ok(GateFix::Defer);
7767        }
7768        let mut record = GateFixRecord {
7769            agent: seat.agent.clone(),
7770            failed,
7771            notes: String::new(),
7772            committed: false,
7773            error: None,
7774        };
7775        match out {
7776            AgentOutcome::Ok(o) => {
7777                // A missing report is not a failed fix: the round is judged
7778                // by the tree below, and the report only carries prose.
7779                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7780                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7781                }
7782            }
7783            AgentOutcome::Dropped(_) => {
7784                record.error = Some("the CLI dropped the stream".to_owned());
7785            }
7786            AgentOutcome::Quota(o) => {
7787                self.state.quota.push(QuotaLoss::from_report(
7788                    seat.key.clone(),
7789                    "gate-fix",
7790                    o.quota.as_ref(),
7791                ));
7792                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7793            }
7794            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7795        }
7796        self.state.seats.insert(seat.key.clone(), seat);
7797        let coordination = self.coordinate(&winner.worktree).await?;
7798        if let Ok(r) = git::rescue_commit(
7799            &winner.worktree,
7800            &format!("magi: gate fix {attempt} (uncommitted work)"),
7801        )
7802        .await
7803        {
7804            self.state.note_withheld("gate-fix", &r.withheld);
7805        }
7806        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7807        record.committed = after != before;
7808        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7809        let note = record.error.clone();
7810        self.state.gate_fixes.push(record);
7811        self.state.save()?;
7812        if !changed {
7813            self.state.event(
7814                "gate",
7815                match note {
7816                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7817                    None => format!("gate-fix round {attempt}: the tree did not change"),
7818                },
7819            );
7820            return Ok(GateFix::Stop);
7821        }
7822        self.state.event(
7823            "gate",
7824            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7825        );
7826
7827        drop(coordination);
7828        let wait = self.repository_wait();
7829        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7830    }
7831
7832    // ---------------------------------------------------------------- merge
7833
7834    /// One read-only rewrite by the summary's author; a text the gate still
7835    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7836    /// applies on their word or on silence. `None` means the run is parked on
7837    /// that question and nothing may be pushed or opened yet.
7838    async fn guarded_pr_message(
7839        &mut self,
7840        winner: &Candidate,
7841        facts: Option<&BranchFacts>,
7842        posting: bool,
7843    ) -> Result<Option<PrMessage>> {
7844        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7845        let head = match final_head.as_deref() {
7846            Some(h) => crate::pr_outcome::FinalHead::At(h),
7847            None => crate::pr_outcome::FinalHead::Unknown,
7848        };
7849        // Only a pull request that is about to be posted pays for a summary.
7850        let summarized = if posting {
7851            Some(self.summarize_change(winner, final_head.as_deref()).await)
7852        } else {
7853            None
7854        };
7855        // A busy shared resource is not a permanent cause: publishing an
7856        // "unavailable" description would stick (a later head move never
7857        // edits the body). Stop before anything is pushed or opened; the
7858        // run is `Blocked` and a resume asks the summary again.
7859        if self.state.resource_block.is_some() {
7860            self.state.event(
7861                "pr-summary",
7862                "a shared resource was busy; nothing was published, resume to summarize again",
7863            );
7864            return Ok(None);
7865        }
7866        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7867            anyhow::bail!(
7868                "the final commit moved during public description generation; retry before publishing"
7869            );
7870        }
7871        let note = match (&summarized, &self.state.change_summary) {
7872            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7873            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7874            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7875            (None, _) => ChangeNote::NotAsked,
7876        };
7877        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7878        if !posting {
7879            let identity = crate::scrub::Identity::current();
7880            return Ok(Some(PrMessage {
7881                title: crate::scrub::scrub(&pr.title, &identity),
7882                body: crate::scrub::scrub(&pr.body, &identity),
7883            }));
7884        }
7885        // This very text was already rejected and asked about: no second
7886        // judge call, rewrite or question, just carry the record on.
7887        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7888        if let Some(known) = self
7889            .state
7890            .github_text
7891            .clone()
7892            .filter(|g| g.fingerprint == raw_fp)
7893        {
7894            return self.settle_withheld(winner, &pr, known).await;
7895        }
7896        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7897        let mut violations =
7898            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7899        let identity = crate::scrub::Identity::current();
7900        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7901            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7902            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7903        {
7904            violations.push(crate::github_text::Violation::SensitiveData);
7905        }
7906        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7907        // only a language violation starts one; sensitive data alone goes
7908        // straight to the owner, with the redacted text as the fallback.
7909        let has_language = violations
7910            .iter()
7911            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7912        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7913            self.state.event(
7914                "github-text",
7915                if has_language {
7916                    format!("description rejected: {violations:?}; requesting one rewrite")
7917                } else {
7918                    format!("description rejected: {violations:?}; asking the owner")
7919                },
7920            );
7921            let mut rewritten = false;
7922            let mut rejected_rewrite = None;
7923            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7924            {
7925                let output = text;
7926                if let Some(title) = summary_title(&output) {
7927                    // Judge the redacted candidate that will actually be posted.
7928                    // Sensitive data alone must not discard a useful rewrite.
7929                    let raw_body = summary_without_title(&output);
7930                    let clean_title = crate::scrub::scrub(&title, &identity);
7931                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7932                    let redacted = clean_title != title || body != raw_body;
7933                    let title = clean_title;
7934                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7935                    if !complete {
7936                        self.state.event(
7937                            "github-text",
7938                            "description rewrite omitted required findings",
7939                        );
7940                    }
7941                    if !body.trim().is_empty() {
7942                        body.push_str("\n\n");
7943                        body.push_str(&public_pr_evidence(&self.state, head));
7944                    }
7945                    let marker = format!("magi:run/{}", self.state.id);
7946                    if !body.trim().is_empty() && !body.contains(&marker) {
7947                        body.push_str(&format!("\n\n{marker}"));
7948                    }
7949                    let rewrite = PrMessage {
7950                        title: title.clone(),
7951                        body: body.clone(),
7952                    };
7953                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7954                    let mut failed = crate::github_text::check_for(
7955                        &self.state.config.graph,
7956                        &title,
7957                        &body,
7958                        redecision,
7959                    );
7960                    if (!crate::github_text::shareable(&title)
7961                        || !crate::github_text::shareable(&body))
7962                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7963                    {
7964                        failed.push(crate::github_text::Violation::SensitiveData);
7965                    }
7966                    if complete
7967                        && !title.trim().is_empty()
7968                        && !body.trim().is_empty()
7969                        && !failed.is_empty()
7970                    {
7971                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7972                            title: title.clone(),
7973                            body: body.clone(),
7974                            failed: failed
7975                                .iter()
7976                                .map(|v| crate::github_text::category(*v).to_owned())
7977                                .collect(),
7978                        });
7979                    }
7980                    if complete
7981                        && !title.trim().is_empty()
7982                        && !raw_body.trim().is_empty()
7983                        && failed.is_empty()
7984                    {
7985                        decision = redecision;
7986                        pr = PrMessage { title, body };
7987                        rewritten = true;
7988                        self.state
7989                            .event("github-text", "description rewrite passed");
7990                        if redacted {
7991                            self.state.event(
7992                                "github-text",
7993                                "sensitive data removed from description rewrite",
7994                            );
7995                        }
7996                    }
7997                }
7998            }
7999            if !rewritten {
8000                self.state.event(
8001                    "github-text",
8002                    "description rewrite unavailable or rejected; asking the owner",
8003                );
8004                if let Some(w) =
8005                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8006                {
8007                    // A different text than the one asked about before: that
8008                    // question no longer describes anything.
8009                    self.retire_github_text_question("the withheld text changed");
8010                    let record = crate::run::GithubTextAsk {
8011                        rewrite: rejected_rewrite,
8012                        fingerprint: raw_fp,
8013                        title: w.title,
8014                        body: w.body,
8015                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8016                        question: None,
8017                        asks: 0,
8018                        resolved: false,
8019                        chosen_title: None,
8020                        choice: None,
8021                    };
8022                    return self.settle_withheld(winner, &pr, record).await;
8023                }
8024            }
8025        }
8026        let (title, body) =
8027            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8028        Ok(Some(PrMessage { title, body }))
8029    }
8030
8031    /// Describe the whole change from its final diff in either run mode.
8032    ///
8033    /// The summary is tied to the final commit and to the merge base of the
8034    /// resolved remote base; a record for any other pair is dropped and the
8035    /// agent asked again, so a description never outlives the diff it was
8036    /// written from. The seat is read-only and the prompt tells it to read the
8037    /// diff and the code around it. `[roles] synthesizer` selects the agents
8038    /// (unset: the default pick), each asked at most once, advancing on an
8039    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8040    /// carries the reason it is unavailable; nothing is invented then.
8041    async fn summarize_change(
8042        &mut self,
8043        winner: &Candidate,
8044        head: Option<&str>,
8045    ) -> std::result::Result<(), String> {
8046        let Some(head) = head else {
8047            return Err("the final commit could not be read".to_owned());
8048        };
8049        let remote = self.state.config.merge.remote.clone();
8050        let tracking = format!("{remote}/{}", self.state.base_branch);
8051        if !git::rev_exists(&winner.worktree, &tracking).await {
8052            return Err(format!("the remote base `{tracking}` could not be read"));
8053        }
8054        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8055            Ok(b) if !b.is_empty() => b,
8056            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8057        };
8058        let evidence = crate::pr_outcome::summary_evidence(
8059            &self.state,
8060            crate::pr_outcome::FinalHead::At(head),
8061        );
8062        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8063        if self
8064            .state
8065            .change_summary
8066            .as_ref()
8067            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8068        {
8069            return Ok(());
8070        }
8071        if self.state.change_summary.take().is_some() {
8072            self.state.event(
8073                "pr-summary",
8074                "the final diff or evidence changed; the earlier change summary was discarded",
8075            );
8076        }
8077        // Save the source snapshot even if no generation agent is available.
8078        if let Err(e) = write_artifact(
8079            &self.state,
8080            &format!("public-description-{fingerprint}.md"),
8081            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8082        ) {
8083            tracing::warn!("could not save public description evidence: {e:#}");
8084        }
8085        let chain = match agent::pick_chain(
8086            &self.state.config.agents,
8087            self.state.config.roles.synthesizer.as_ref(),
8088            &agent::installed,
8089            "synthesizer",
8090        ) {
8091            Ok(chain) => chain,
8092            Err(e) => {
8093                tracing::warn!("no agent for the change summary: {e:#}");
8094                self.state.event("pr-summary", "no summary agent available");
8095                return Err("no agent was available to summarize it".to_owned());
8096            }
8097        };
8098        let subjects = git::subjects(&winner.worktree, &base, head)
8099            .await
8100            .unwrap_or_default();
8101        let stat = git::diff_stat(&winner.worktree, &base, head)
8102            .await
8103            .unwrap_or_default();
8104        let stat: String = stat.chars().take(6_000).collect();
8105        let mut prompt = crate::prompt::change_summary_in(
8106            &base,
8107            head,
8108            &subjects,
8109            stat.trim(),
8110            &self.state.config.graph.github_language,
8111        );
8112        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8113        let sessions = self.state.config.graph.sessions;
8114        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8115        let prompts = self.state.config.prompts.clone();
8116        let cache = self.state.config.cache_dir();
8117        let run = self.state.id.clone();
8118        for (i, spec) in chain.into_iter().enumerate() {
8119            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8120            let github_spec = match agent::for_github_text(
8121                &spec,
8122                &self.state.config.graph.github_language,
8123                &winner.worktree,
8124            ) {
8125                Ok(spec) => spec,
8126                Err(e) => {
8127                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8128                    continue;
8129                }
8130            };
8131            let job = SeatJob {
8132                spec: github_spec,
8133                seat,
8134                cwd: winner.worktree.clone(),
8135                prompt: prompt.clone(),
8136                timeout,
8137                allow_write: false,
8138                sessions,
8139                artifacts: agent::artifacts_dir(&self.state.dir()),
8140                stem: if i == 0 {
8141                    "change-summary".to_owned()
8142                } else {
8143                    format!("change-summary-{}", spec.id)
8144                },
8145                handover: None,
8146            };
8147            let ctx = WaveCtx {
8148                run: &run,
8149                node: "pr-summary",
8150                prompts: &prompts,
8151                cache: cache.as_deref(),
8152                round: None,
8153                carry_seats: false,
8154                pauses: Some((&self.pause, &self.interrupt)),
8155            };
8156            let (seat, outcome) =
8157                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8158            self.state.seats.insert(seat.key.clone(), seat);
8159            let text = match outcome {
8160                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8161                AgentOutcome::ResourceBlocked(why) => {
8162                    // No agent was asked: the run stops resumably before
8163                    // anything is published (see `guarded_pr_message`).
8164                    if self.state.resource_block.is_none() {
8165                        self.state.resource_block = Some(crate::run::ResourceBlock {
8166                            cause: crate::run::ResourceBlockCause::Other,
8167                            node: "pr-summary".to_owned(),
8168                            kind: "shared resource".to_owned(),
8169                            detail: why,
8170                            at: Timestamp::now(),
8171                        });
8172                    }
8173                    return Err("a shared resource was busy".to_owned());
8174                }
8175                _ => continue,
8176            };
8177            let blind = &self.state.config.blind;
8178            let Some(title) = summary_title(&text) else {
8179                continue;
8180            };
8181            let body: String = summary_without_title(&text)
8182                .lines()
8183                .filter(|l| !l.contains("magi:run/"))
8184                .collect::<Vec<_>>()
8185                .join("\n");
8186            let body = blind::sanitize_prose(body.trim(), blind);
8187            let title = blind::sanitize_prose(&title, blind);
8188            if body.trim().is_empty() || title.trim().is_empty() {
8189                continue;
8190            }
8191            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8192                continue;
8193            }
8194            // The branch must still be the one the agent was shown.
8195            if git::rev_parse(&winner.worktree, "HEAD")
8196                .await
8197                .ok()
8198                .as_deref()
8199                != Some(head)
8200            {
8201                return Err("the branch moved while the summary was written".to_owned());
8202            }
8203            if git::merge_base(&winner.worktree, &tracking, head)
8204                .await
8205                .ok()
8206                .as_deref()
8207                != Some(base.as_str())
8208                || crate::pr_outcome::summary_evidence(
8209                    &self.state,
8210                    crate::pr_outcome::FinalHead::At(head),
8211                ) != evidence
8212            {
8213                return Err("the summary evidence changed while it was written".to_owned());
8214            }
8215            self.state.change_summary = Some(crate::run::ChangeSummary {
8216                head: head.to_owned(),
8217                base,
8218                evidence: fingerprint,
8219                title,
8220                body,
8221            });
8222            self.state.event(
8223                "pr-summary",
8224                format!("change summary written by {} for the final diff", spec.id),
8225            );
8226            if let Err(e) = self.state.save() {
8227                tracing::warn!("could not save the change summary: {e:#}");
8228            }
8229            return Ok(());
8230        }
8231        self.state
8232            .event("pr-summary", "no agent produced a change summary");
8233        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8234    }
8235
8236    /// One read-only rewrite request for a withheld description. The
8237    /// implementer's own conversation is tried first when it exists and can be
8238    /// resumed; then the `[roles] language_judge` chain (or the default pick)
8239    /// writes it in a separate conversation, skipping the implementer's agent.
8240    /// The prompt carries the whole text, so no agent needs any memory. Each
8241    /// agent is asked at most once and the chain advances on an error, quota,
8242    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
8243    /// that has run out of quota must not end the rewrite. Nothing is recorded
8244    /// as a handover.
8245    async fn rewrite_pr_text(
8246        &mut self,
8247        winner: &Candidate,
8248        pr: &PrMessage,
8249        violations: &[crate::github_text::Violation],
8250    ) -> Option<String> {
8251        let prompt = format!(
8252            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8253            pr.title, pr.body
8254        );
8255        let sessions = self.state.config.graph.sessions;
8256        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8257        // (spec, the resumed implementer's seat). A fallback seat is created
8258        // only when its turn comes, so an unused fallback leaves no record.
8259        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8260        let key = format!("impl-{}", winner.label);
8261        if let Some(spec) = self
8262            .state
8263            .config
8264            .agents
8265            .iter()
8266            .find(|a| a.id == winner.agent)
8267            .cloned()
8268            && let Some(existing) = self.state.seats.get(&key)
8269            && existing.agent == spec.id
8270            && agent::has_session(spec.kind, existing, sessions)
8271        {
8272            attempts.push((spec, Some(existing.clone())));
8273        }
8274        match agent::pick_chain(
8275            &self.state.config.agents,
8276            self.state.config.roles.language_judge.as_ref(),
8277            &agent::installed,
8278            "language_judge",
8279        ) {
8280            Ok(chain) => {
8281                for spec in chain {
8282                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8283                        continue;
8284                    }
8285                    attempts.push((spec, None));
8286                }
8287            }
8288            Err(e) if attempts.is_empty() => {
8289                tracing::warn!("no agent for the description rewrite: {e:#}");
8290                self.state
8291                    .event("github-text", "no rewrite agent available");
8292                return None;
8293            }
8294            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8295        }
8296        let prompts = self.state.config.prompts.clone();
8297        let cache = self.state.config.cache_dir();
8298        let run = self.state.id.clone();
8299        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8300            let resumed = resumed_seat.is_some();
8301            let seat = resumed_seat.unwrap_or_else(|| {
8302                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8303            });
8304            let github_spec = match agent::for_github_text(
8305                &spec,
8306                &self.state.config.graph.github_language,
8307                &winner.worktree,
8308            ) {
8309                Ok(spec) => spec,
8310                Err(e) => {
8311                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8312                    continue;
8313                }
8314            };
8315            let job = SeatJob {
8316                spec: github_spec,
8317                seat,
8318                cwd: winner.worktree.clone(),
8319                prompt: prompt.clone(),
8320                timeout: retry_budget(timeout_full, resumed),
8321                allow_write: false,
8322                sessions,
8323                artifacts: agent::artifacts_dir(&self.state.dir()),
8324                stem: if i == 0 {
8325                    "github-text-rewrite".to_owned()
8326                } else {
8327                    format!("github-text-rewrite-{}", spec.id)
8328                },
8329                handover: None,
8330            };
8331            let ctx = WaveCtx {
8332                run: &run,
8333                node: "github-text",
8334                prompts: &prompts,
8335                cache: cache.as_deref(),
8336                round: None,
8337                carry_seats: false,
8338                pauses: Some((&self.pause, &self.interrupt)),
8339            };
8340            let (seat, outcome) =
8341                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8342            self.state.seats.insert(seat.key.clone(), seat);
8343            match outcome {
8344                AgentOutcome::Ok(o)
8345                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8346                {
8347                    return Some(o.text);
8348                }
8349                AgentOutcome::ResourceBlocked(_) => return None,
8350                _ => {}
8351            }
8352        }
8353        None
8354    }
8355
8356    /// Abandon the standing posting-gate question, if any.
8357    fn retire_github_text_question(&mut self, why: &str) {
8358        let Some(id) = self
8359            .state
8360            .github_text
8361            .as_ref()
8362            .and_then(|g| g.question.clone())
8363        else {
8364            return;
8365        };
8366        let _ = ask::Questions::open().update(&id, |q| {
8367            q.abandon(why);
8368            Ok(())
8369        });
8370    }
8371
8372    /// Carry a withheld text to its end: file the question, park while it is
8373    /// open, and on an answer, silence or timeout pick the text to post. The
8374    /// decision is saved before it is used, so it applies once.
8375    async fn settle_withheld(
8376        &mut self,
8377        winner: &Candidate,
8378        pr: &PrMessage,
8379        mut g: crate::run::GithubTextAsk,
8380    ) -> Result<Option<PrMessage>> {
8381        use crate::github_text as gt;
8382        let store = ask::Questions::open();
8383        let timeout = self.state.config.graph.answer_timeout;
8384        if !g.resolved {
8385            let mut standing = g
8386                .question
8387                .as_ref()
8388                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8389            if let Some(q) = standing.as_ref()
8390                && q.status.open()
8391                && gt::expired(q, timeout)
8392            {
8393                let why = format!("no answer within {}s of asking", timeout.max(1));
8394                standing = store
8395                    .update(&q.id, |q| {
8396                        q.abandon(&why);
8397                        Ok(())
8398                    })
8399                    .ok()
8400                    .map(|(q, ())| q);
8401            }
8402            let mut ask_again: Option<Option<Vec<String>>> = None;
8403            match standing {
8404                // Recorded but never written (a stop between the two saves),
8405                // or removed: ask again without spending the retry.
8406                None if g.asks <= 1 => {
8407                    g.asks = 0;
8408                    ask_again = Some(None);
8409                }
8410                None => {
8411                    g.resolved = true;
8412                    g.chosen_title = None;
8413                    g.choice = Some(GithubTextChoice::Fallback);
8414                }
8415                Some(q) if q.status.open() => {
8416                    self.park_on_github_text(&g)?;
8417                    return Ok(None);
8418                }
8419                Some(q) => match gt::read_reply(&q) {
8420                    gt::Reply::Fallback => {
8421                        g.resolved = true;
8422                        g.chosen_title = None;
8423                        g.choice = Some(GithubTextChoice::Fallback);
8424                    }
8425                    gt::Reply::Original => {
8426                        g.resolved = true;
8427                        g.chosen_title = None;
8428                        g.choice = Some(GithubTextChoice::Original);
8429                    }
8430                    gt::Reply::Rewrite => {
8431                        g.resolved = true;
8432                        g.chosen_title = None;
8433                        g.choice = Some(if g.rewrite.is_some() {
8434                            GithubTextChoice::Rewrite
8435                        } else {
8436                            GithubTextChoice::Fallback
8437                        });
8438                    }
8439                    gt::Reply::Title(text) => {
8440                        let first = text
8441                            .lines()
8442                            .map(str::trim)
8443                            .find(|l| !l.is_empty())
8444                            .unwrap_or("");
8445                        let decision = if first.is_empty() {
8446                            None
8447                        } else {
8448                            let candidate = PrMessage {
8449                                title: first.to_owned(),
8450                                body: String::new(),
8451                            };
8452                            self.judge_pr_language(&winner.worktree, &candidate).await
8453                        };
8454                        match gt::vet_title(
8455                            &text,
8456                            decision,
8457                            self.state.config.graph.github_english(),
8458                        ) {
8459                            Ok(title) => {
8460                                g.resolved = true;
8461                                g.chosen_title = Some(title);
8462                                g.choice = Some(GithubTextChoice::MyTitle);
8463                            }
8464                            Err(cats) if g.asks < 2 => {
8465                                ask_again = Some(Some(
8466                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8467                                ))
8468                            }
8469                            Err(_) => {
8470                                g.resolved = true;
8471                                g.chosen_title = None;
8472                                g.choice = Some(GithubTextChoice::Fallback);
8473                            }
8474                        }
8475                    }
8476                },
8477            }
8478            if let Some(rejected) = ask_again {
8479                // A retry (after a rejected replacement title) is still about
8480                // the original text: the same ORIGINAL / FALLBACK comparison.
8481                let w = gt::Withheld {
8482                    title: g.title,
8483                    body: g.body,
8484                    categories: g.categories.clone(),
8485                };
8486                // Named by the original text's fingerprint alone, so a resume
8487                // rewrites the same file and the detail stays identical.
8488                let artifact = match crate::run::write_artifact(
8489                    &self.state,
8490                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
8491                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8492                ) {
8493                    Ok(p) => Some(p),
8494                    Err(e) => {
8495                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
8496                        None
8497                    }
8498                };
8499                let (detail, comparison) = gt::question_content(
8500                    &w,
8501                    &pr.title,
8502                    &pr.body,
8503                    &self.state.id,
8504                    rejected.as_deref(),
8505                    artifact.as_deref(),
8506                    g.rewrite.as_ref(),
8507                );
8508                let mut q = ask::Question::new(
8509                    self.state.id.clone(),
8510                    gt::ASK_NODE.to_owned(),
8511                    gt::ASK_SEAT.to_owned(),
8512                    gt::question_summary(&self.state.config.graph.language, &w),
8513                    detail,
8514                    gt::question_choices(&w, g.rewrite.as_ref()),
8515                );
8516                q.posting_comparison = Some(comparison);
8517                q.answer_timeout = timeout;
8518                // Recorded before it is written: a stop in between leaves a
8519                // record naming a question the resume then files again.
8520                g.question = Some(q.id.clone());
8521                g.asks += 1;
8522                self.state.github_text = Some(g.clone());
8523                self.state.event(
8524                    "github-text",
8525                    format!("asking the owner about the withheld text ({})", q.short()),
8526                );
8527                self.state.save()?;
8528                store
8529                    .put_executable(&mut q)
8530                    .context("file the posting-gate question")?;
8531                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8532                    tracing::warn!(
8533                        "could not notify about posting-gate question {}: {e:#}",
8534                        q.short()
8535                    );
8536                }
8537                self.park_on_github_text(&g)?;
8538                return Ok(None);
8539            }
8540        }
8541        let resolved = g.chosen_title.clone();
8542        let original = g.choice == Some(GithubTextChoice::Original);
8543        // Field names only: the event log never records the text.
8544        self.state.event(
8545            "github-text",
8546            if original {
8547                format!(
8548                    "posting the original text for: {}",
8549                    match (g.title, g.body) {
8550                        (true, true) => "title, description",
8551                        (true, false) => "title",
8552                        _ => "description",
8553                    }
8554                )
8555            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8556                "posting the saved rewrite title and description".to_owned()
8557            } else if resolved.is_some() {
8558                "posting the owner's replacement title".to_owned()
8559            } else {
8560                "posting the neutral text for the withheld fields".to_owned()
8561            },
8562        );
8563        self.state.github_text = Some(g.clone());
8564        self.state.save()?;
8565        if g.choice == Some(GithubTextChoice::Rewrite)
8566            && let Some(r) = g.rewrite.as_ref()
8567        {
8568            return Ok(Some(PrMessage {
8569                title: r.title.clone(),
8570                body: r.body.clone(),
8571            }));
8572        }
8573        let w = gt::Withheld {
8574            title: g.title,
8575            body: g.body,
8576            categories: g.categories.clone(),
8577        };
8578        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8579        if let Some(t) = resolved {
8580            title = t;
8581        }
8582        // Every withheld field is now neutral or vetted English.
8583        let english = gt::LanguageDecision {
8584            title_english: true,
8585            body_english: true,
8586            body_complete: true,
8587        };
8588        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8589        if original {
8590            // The owner's explicit choice overrides the gate for the fields it
8591            // was made for; the others keep what the gate produced.
8592            if g.title {
8593                title = pr.title.clone();
8594            }
8595            if g.body {
8596                body = pr.body.clone();
8597            }
8598        }
8599        Ok(Some(PrMessage { title, body }))
8600    }
8601
8602    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8603        self.state.github_text = Some(g.clone());
8604        self.state.end_pass_parked(ParkCause::Question);
8605        self.state.parked = true;
8606        self.state.event(
8607            "github-text",
8608            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8609        );
8610        self.state.save()
8611    }
8612
8613    /// Ask `[roles] language_judge` about this text and record which source
8614    /// decided; the text itself is never recorded.
8615    async fn judge_pr_language(
8616        &mut self,
8617        cwd: &Path,
8618        pr: &PrMessage,
8619    ) -> Option<crate::github_text::LanguageDecision> {
8620        self.state.config.roles.language_judge.as_ref()?;
8621        if !self.state.config.graph.language_gate() {
8622            return None;
8623        }
8624        let decision =
8625            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8626        self.state.event(
8627            "github-text",
8628            if decision.is_some() {
8629                "language decided by the language judge"
8630            } else {
8631                "language judge unavailable; using built-in heuristics"
8632            },
8633        );
8634        decision
8635    }
8636
8637    async fn merge(&mut self) -> Result<()> {
8638        if self.resource_halt()? || self.hold_base_sync().await? {
8639            return Ok(());
8640        }
8641        // Same reasoning as `gate`: ask the review and gate records directly
8642        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8643        // skip can rewrite on reentry to something that no longer says
8644        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8645        // so a hand-off (open findings, green verification) reaches merge
8646        // exactly like a genuinely clean round does.
8647        //
8648        // A run resumed mid-`land` never reaches here at all: `execute`
8649        // recognises `RunStatus::Landing` before it even calls `prep`, and
8650        // routes straight to `run_land` instead. That has to happen a level
8651        // up from this function, not with a check in here, because
8652        // `review_loop`'s own status recomputation (see its doc) runs
8653        // *before* `merge` on every reentry and would otherwise overwrite
8654        // the `Landing` marker with `Gating` before this node ever saw it.
8655        if self
8656            .state
8657            .base_sync
8658            .as_ref()
8659            .is_some_and(|s| s.conflict.is_some())
8660            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8661                != Some(RunStatus::Gating)
8662            // `gate_ran == false` is not "passed" - `gate` leaves it false
8663            // both before it has ever run and when its last attempt was
8664            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8665            // permission to merge on nothing but the review record. Only a
8666            // gate that actually ran may proceed; an intentional local
8667            // review can also finish with verification explicitly not configured.
8668            // `RunState::gate_status` computes that distinction.
8669            || !self.state.gate_allows_progress()
8670        {
8671            return Ok(());
8672        }
8673        // This node's own record, not `status`: `status == Ready` is not
8674        // unique to the harmless `MergeMode::None` path this line was
8675        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8676        // run's PR was closed without merging — and on that run `mode` is
8677        // still `Pr`, so a reentry that fell through here would push and
8678        // open a second pull request. `self.state.merge` is set exactly once
8679        // this node (or `land`) has already produced a verdict, under every
8680        // mode, which is what "already done" actually means here.
8681        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8682            return Ok(());
8683        }
8684        let Some(winner) = self.state.winner().cloned() else {
8685            return Ok(());
8686        };
8687        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8688        if self.state.gate_head.as_deref() != Some(head.as_str()) {
8689            self.state.status = RunStatus::Blocked;
8690            self.state.event(
8691                "gate",
8692                "winner changed after gate; fresh verification required",
8693            );
8694            self.state.save()?;
8695            return Ok(());
8696        }
8697        let wait = self.repository_wait();
8698        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8699            return Ok(());
8700        }
8701        let repo = self.state.repo.clone();
8702        let base = self.state.base_branch.clone();
8703        let mode = self.state.config.merge.mode;
8704        let style = self.state.config.merge.style;
8705        let facts = if is_review_run(&self.state) {
8706            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8707            let start = review_base(
8708                &repo,
8709                &self.state.config.merge.remote,
8710                &base,
8711                &self.state.base_commit,
8712                &winner.branch,
8713            )
8714            .await;
8715            branch_facts(&repo, &start, &winner.branch).await
8716        } else {
8717            None
8718        };
8719        // `None` when the base could not be freshly read: then an adopted
8720        // pull request's title is left alone.
8721        let leaked = if is_review_run(&self.state) {
8722            leaked_subjects(&self.state, &winner.branch).await
8723        } else {
8724            Some(Vec::new())
8725        };
8726        // `None`: parked on the owner's word about a withheld title or body.
8727        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8728        // the resume arrives here again.
8729        let Some(pr) = self
8730            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8731            .await?
8732        else {
8733            return Ok(());
8734        };
8735        let message = pr.commit_message();
8736
8737        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
8738        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
8739            return Ok(());
8740        }
8741        let outcome = match mode {
8742            MergeMode::None => MergeOutcome {
8743                mode,
8744                ok: true,
8745                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8746                empty: false,
8747            },
8748            MergeMode::Pr | MergeMode::Local
8749                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8750            {
8751                MergeOutcome {
8752                    mode,
8753                    ok: false,
8754                    detail: empty_candidate_detail(&self.state, &base),
8755                    empty: true,
8756                }
8757            }
8758            MergeMode::Local => {
8759                let on = git::current_branch(&repo).await?;
8760                if on.as_deref() != Some(base.as_str()) {
8761                    MergeOutcome {
8762                        mode,
8763                        ok: false,
8764                        detail: format!(
8765                            "{} has {} checked out, not the base branch {base}",
8766                            repo.display(),
8767                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8768                        ),
8769                        empty: false,
8770                    }
8771                } else if !git::is_clean(&repo).await? {
8772                    MergeOutcome {
8773                        mode,
8774                        ok: false,
8775                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8776                        empty: false,
8777                    }
8778                } else {
8779                    let out = match style {
8780                        MergeStyle::Merge => {
8781                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8782                        }
8783                        MergeStyle::Squash => {
8784                            git::merge_squash(&repo, &winner.branch, &message).await?
8785                        }
8786                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8787                    };
8788                    MergeOutcome {
8789                        mode,
8790                        ok: out.ok(),
8791                        detail: if out.ok() { out.stdout } else { out.stderr },
8792                        empty: false,
8793                    }
8794                }
8795            }
8796            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8797                self.state.pr_publication_pending = true;
8798                self.state.save()?;
8799                let out =
8800                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8801                self.state.pr_publication_pending = out.is_err();
8802                MergeOutcome {
8803                    mode,
8804                    ok: out.is_ok(),
8805                    detail: match out {
8806                        Ok(url) => url,
8807                        Err(e) => format!("{e:#}"),
8808                    },
8809                    empty: false,
8810                }
8811            }
8812            MergeMode::Pr => {
8813                let remote = self.state.config.merge.remote.clone();
8814                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8815                let pushed = if self.state.foreign_checkout.is_some() {
8816                    let refspec = format!("{delivery_head}:refs/heads/{public}");
8817                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
8818                } else {
8819                    git::push(&winner.worktree, &remote, &winner.branch).await?
8820                };
8821                if !pushed.ok() {
8822                    MergeOutcome {
8823                        mode,
8824                        ok: false,
8825                        detail: pushed.stderr,
8826                        empty: false,
8827                    }
8828                } else {
8829                    if self.state.foreign_checkout.is_some() {
8830                        self.state.pr_pushed_head = Some(delivery_head.clone());
8831                    }
8832                    // A retry or resume of a run whose branch already has an
8833                    // open pull request adopts it rather than failing on a
8834                    // duplicate. Only this winner branch into this base:
8835                    // `branch_for` derives the name from the run id, so a
8836                    // different run's pull request never matches.
8837                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
8838                    let out = match pr_merge_plan(found) {
8839                        PrPlan::Create => {
8840                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
8841                                .await
8842                        }
8843                        PrPlan::Adopt { url, title } => {
8844                            self.state
8845                                .event("merge", format!("Pr: adopted open pull request {url}"));
8846                            if self.state.github_text.as_ref().is_some_and(|g| {
8847                                g.resolved
8848                                    && g.choice == Some(GithubTextChoice::Rewrite)
8849                                    && g.rewrite.is_some()
8850                            }) {
8851                                // The owner chose the entire pair, even if the
8852                                // existing title already matches or was edited.
8853                                land::set_pr_text_verbatim(
8854                                    &winner.worktree,
8855                                    &url,
8856                                    &pr.title,
8857                                    &pr.body,
8858                                )
8859                                .await
8860                                .map(|()| url)
8861                            } else {
8862                                if title != pr.title
8863                                    && (!is_review_run(&self.state)
8864                                        || leaked
8865                                            .as_deref()
8866                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8867                                    && let Err(e) =
8868                                        if self.state.github_text.as_ref().is_some_and(|g| {
8869                                            g.resolved
8870                                                && g.title
8871                                                && g.choice == Some(GithubTextChoice::Original)
8872                                        }) {
8873                                            // The owner chose to post this title unmodified.
8874                                            land::set_pr_title_verbatim(
8875                                                &winner.worktree,
8876                                                &url,
8877                                                &pr.title,
8878                                            )
8879                                            .await
8880                                        } else {
8881                                            land::set_pr_title(
8882                                                &mut self.state,
8883                                                &winner.worktree,
8884                                                &url,
8885                                                &pr.title,
8886                                            )
8887                                            .await
8888                                        }
8889                                {
8890                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8891                                    self.state
8892                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8893                                }
8894                                Ok(url)
8895                            }
8896                        }
8897                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8898                    };
8899                    match out {
8900                        Ok(url) => MergeOutcome {
8901                            mode,
8902                            ok: true,
8903                            detail: url,
8904                            empty: false,
8905                        },
8906                        Err(e) => MergeOutcome {
8907                            mode,
8908                            ok: false,
8909                            detail: e.to_string(),
8910                            empty: false,
8911                        },
8912                    }
8913                }
8914            }
8915        };
8916
8917        self.state.status = match (mode, outcome.ok) {
8918            (MergeMode::None, _) => RunStatus::Ready,
8919            (_, true) if outcome.empty => RunStatus::Ready,
8920            (MergeMode::Pr, true) => RunStatus::Ready,
8921            (_, true) => RunStatus::Merged,
8922            (_, false) => RunStatus::Blocked,
8923        };
8924        self.state.event(
8925            "merge",
8926            format!(
8927                "{:?}: {}",
8928                mode,
8929                outcome.detail.lines().next().unwrap_or("")
8930            ),
8931        );
8932        self.state.merge = Some(outcome);
8933        self.state.save()?;
8934
8935        // The PR is open and the run would historically stop here, leaving the
8936        // operator to watch checks, feed review comments back to a fixer, and
8937        // merge. That was done by hand six times in one session before this
8938        // existed. Opt-in, because merging is the one irreversible thing magi
8939        // can do to a repository.
8940        if self.state.config.graph.land
8941            && mode == MergeMode::Pr
8942            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8943        {
8944            self.run_land().await?;
8945        }
8946        // `run_land` may have left `status` at `Landing` - still waiting on
8947        // CI or the owner's approval, not actually settled - so this has to
8948        // read whatever `status` ended up as here, not the `Merged` this
8949        // function set a few lines up.
8950        self.settle_questions();
8951        Ok(())
8952    }
8953
8954    /// Enter `land`.
8955    ///
8956    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8957    /// a resumed run's re-entry. `land::land` itself is what serialises the
8958    /// two git-mutating moments inside the loop — the rebase push and
8959    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8960    /// to hold a lock across the whole call, and doing so would serialise
8961    /// this run's CI wait against a *different* run's land-approval resume
8962    /// in the same repository, which is exactly the "must not wait on
8963    /// another task" property the daemon's slot-freeing exists to give.
8964    async fn run_land(&mut self) -> Result<()> {
8965        crate::pr_head::ensure_owner(&self.state)?;
8966        let url = self
8967            .state
8968            .pr
8969            .as_ref()
8970            .map(|pr| pr.url.clone())
8971            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8972            .unwrap_or_default();
8973        let url = url.lines().next().unwrap_or("").trim().to_owned();
8974        if !url.starts_with("http") {
8975            return Ok(());
8976        }
8977        // A land failure is not a lost run: the work is on a branch and the
8978        // pull request is open, which is exactly where a human takes over.
8979        loop {
8980            match land::land(&mut self.state, &url).await {
8981                Ok(_)
8982                    if self
8983                        .state
8984                        .merge_repairs
8985                        .last()
8986                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8987                {
8988                    if self.repair_merge_review().await? {
8989                        continue;
8990                    }
8991                }
8992                Ok(_) if self.state.resource_block.is_some() => {
8993                    return self.stop_for_resource();
8994                }
8995                Ok(pr) if self.state.parked => {
8996                    // `land` already saved the parked marker; nothing here
8997                    // overrides `status` back to a terminal value while an
8998                    // approval is still outstanding.
8999                    let _ = pr;
9000                }
9001                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9002                    self.state.status = RunStatus::Superseded;
9003                    self.state.save()?;
9004                }
9005                Ok(pr) => {
9006                    self.state.status = match pr.state {
9007                        land::PrLifecycle::Merged => RunStatus::Merged,
9008                        _ => RunStatus::Blocked,
9009                    };
9010                    // Downstream of a confirmed merge only - see
9011                    // `bump::should_release_bump`'s own doc for why this one
9012                    // check covers all three of `land`'s success paths.
9013                    // Best-effort: the run already landed, so a failure here
9014                    // (the decision call, `gh`, `cargo`) is recorded and never
9015                    // turns a landed run into a failed one.
9016                    if bump::should_release_bump(self.state.status)
9017                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9018                    {
9019                        // The event is the run's own record. Not-eligible cases
9020                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9021                        // `Err` is a bump that was tried and failed:
9022                        // `after_merge` itself raises the operator notice for
9023                        // that, whether or not a release PR exists yet.
9024                        self.state
9025                            .event("bump", format!("release bump skipped: {e:#}"));
9026                    }
9027                    // Independent of the bump, and best-effort in the same way:
9028                    // findings the merge left open become follow-up tasks.
9029                    if self.state.status == RunStatus::Merged {
9030                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9031                    }
9032                    self.state.save()?;
9033                }
9034                Err(e) => {
9035                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9036                        return Err(e);
9037                    }
9038                    self.state.status = RunStatus::Blocked;
9039                    self.state.event("land", format!("gave up: {e}"));
9040                    self.state.save()?;
9041                }
9042            }
9043            break;
9044        }
9045        Ok(())
9046    }
9047
9048    /// One repair cycle stays under the existing driver/task claim. An
9049    /// interrupted fixer is rolled back and held rather than run twice.
9050    async fn repair_merge_review(&mut self) -> Result<bool> {
9051        use crate::run::MergeRepairPhase as Phase;
9052        crate::pr_head::ensure_owner(&self.state)?;
9053        let i = self.state.merge_repairs.len() - 1;
9054        let request = self.state.merge_repairs[i].clone();
9055        if request.phase == Phase::Fixing {
9056            let restored = match self.state.winner().cloned() {
9057                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9058                None => Err(anyhow::anyhow!("repair has no winning branch")),
9059            };
9060            let why = match restored {
9061                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9062                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9063                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9064            };
9065            self.hold_merge_repair(i, why)?;
9066            return Ok(false);
9067        }
9068        // A completed repair gate is a publication checkpoint. On reentry,
9069        // reject external changes before the ordinary gate path can replace
9070        // that evidence by verifying a new head. Requested repairs still
9071        // start their own review generation below.
9072        if request.phase == Phase::Reviewing && self.state.gate_ran {
9073            let winner = self
9074                .state
9075                .winner()
9076                .context("repair has no winning branch")?;
9077            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9078            if self.state.gate_head.as_deref() != Some(head.as_str())
9079                || !git::is_clean(&winner.worktree).await?
9080            {
9081                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9082                return Ok(false);
9083            }
9084        }
9085        let failure = if i >= self.state.config.graph.land_rounds
9086            || self.state.config.graph.review_rounds == 0
9087        {
9088            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9089        } else {
9090            None
9091        };
9092        if let Some(why) = failure {
9093            self.hold_merge_repair(i, why)?;
9094            return Ok(false);
9095        }
9096        if request.phase == Phase::Requested {
9097            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9098                Ok(pr) => pr,
9099                Err(e) => {
9100                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9101                    return Ok(false);
9102                }
9103            };
9104            self.state.merge_repairs[i].phase = Phase::Fixing;
9105            self.state.land_approval = None;
9106            self.state.land_armed_head = None;
9107            self.state.save()?;
9108            // Serialize the exact records, including finding ids/details,
9109            // reconsideration reasons, final votes and verification output.
9110            let evidence = serde_json::to_string_pretty(&request)?;
9111            let reason = format!(
9112                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9113            );
9114            let budget = self.state.config.graph.land_rounds;
9115            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9116            match result {
9117                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9118                Ok(land::Fixed::Deferred) => {
9119                    // An unavailable fixer did not spend this repair request.
9120                    // Restore partial fallback work before retrying it, as for
9121                    // an interrupted repair, without closing the request.
9122                    let worktree = self
9123                        .state
9124                        .winner()
9125                        .context("repair has no winning branch")?
9126                        .worktree
9127                        .clone();
9128                    self.graft_restore(&worktree, &request.head).await?;
9129                    self.state.merge_repairs[i].phase = Phase::Requested;
9130                    self.stop_for_resource()?;
9131                    return Ok(false);
9132                }
9133                Ok(land::Fixed::Failed(why)) => {
9134                    self.hold_merge_repair(
9135                        i,
9136                        format!(
9137                            "repair failed: {why}; inspect the branch and run an explicit review"
9138                        ),
9139                    )?;
9140                    return Ok(false);
9141                }
9142                Err(e) => {
9143                    self.hold_merge_repair(
9144                        i,
9145                        format!(
9146                            "repair failed: {e:#}; inspect the branch and run an explicit review"
9147                        ),
9148                    )?;
9149                    return Ok(false);
9150                }
9151            }
9152            // Even no change gets a real new reviewer wave. The previous
9153            // evidence remains in the request; a former clean/concluded round
9154            // cannot short-circuit the new review or gate.
9155            self.begin_merge_repair_review(i)?;
9156        }
9157        self.review_loop().await?;
9158        self.gate().await?;
9159        if self.resource_halt()? || self.state.parked {
9160            return Ok(false);
9161        }
9162        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9163            != Some(RunStatus::Gating)
9164            || !self.state.gate_allows_progress()
9165        {
9166            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9167            return Ok(false);
9168        }
9169        // Review and gate may have added commits after the initial repair.
9170        // Publish exactly their final head to the existing PR, with a lease.
9171        let winner = self
9172            .state
9173            .winner()
9174            .cloned()
9175            .context("repair has no winning branch")?;
9176        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9177        if self.state.gate_head.as_deref() != Some(after.as_str())
9178            || !git::is_clean(&winner.worktree).await?
9179        {
9180            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9181            return Ok(false);
9182        }
9183        let expected = self
9184            .state
9185            .pr_pushed_head
9186            .as_deref()
9187            .unwrap_or(&request.head)
9188            .to_owned();
9189        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9190        // Push the checked commit itself: a local ref moving after the check
9191        // must not change which commit the forge receives.
9192        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9193            return Ok(false);
9194        }
9195        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9196        let refspec = format!("{after}:refs/heads/{public}");
9197        let push = git::git_raw(
9198            &winner.worktree,
9199            &["push", &lease, &self.state.config.merge.remote, &refspec],
9200        )
9201        .await;
9202        match push {
9203            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9204            Ok(out) => {
9205                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9206                return Ok(false);
9207            }
9208            Err(e) => {
9209                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9210                return Ok(false);
9211            }
9212        }
9213        self.state.merge_repairs[i].phase = Phase::Complete;
9214        self.state.status = RunStatus::Landing;
9215        self.state.event(
9216            "repair",
9217            "new review and gate completed; fresh owner merge decision required",
9218        );
9219        self.state.save()?;
9220        Ok(true)
9221    }
9222
9223    fn merge_repair_artifacts(&self) -> PathBuf {
9224        let artifacts = agent::artifacts_dir(&self.state.dir());
9225        match self
9226            .state
9227            .merge_repairs
9228            .last()
9229            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9230        {
9231            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9232            None => artifacts,
9233        }
9234    }
9235
9236    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9237        if let Some(e2e) = self.state.head_e2e.take() {
9238            self.state
9239                .verification_history
9240                .push(crate::run::VerificationEvidence {
9241                    superseded_by: None,
9242                    archived_at: Some(Timestamp::now()),
9243                    fixes_spent: Some(self.state.gate_fixes.len()),
9244                    gate_head: None,
9245                    gate_ran: false,
9246                    gate: Vec::new(),
9247                    e2e: Some(e2e),
9248                });
9249        }
9250        self.state.reviews.clear();
9251        self.state.gate.clear();
9252        self.state.gate_ran = false;
9253        self.state.gate_head = None;
9254        self.state.gate_fixes.clear();
9255        self.state.gate_repair_e2e = None;
9256        self.state.pre_gate.clear();
9257        self.state.pre_gate_commit = None;
9258        self.state.contested_handoff = None;
9259        self.state.land_reruns = None;
9260        self.state.land_approval = None;
9261        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9262        self.state.status = RunStatus::Reviewing;
9263        self.state.save()
9264    }
9265
9266    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9267        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9268        self.state.merge_repairs[index].failure = Some(why.clone());
9269        self.state.status = RunStatus::Blocked;
9270        self.state.event("repair", why);
9271        self.state.save()
9272    }
9273
9274    // -------------------------------------------------------------- helpers
9275
9276    /// Fetch or create a seat, keeping its conversation across nodes.
9277    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9278        if let Some(existing) = self.state.seats.get(key)
9279            && existing.agent == agent
9280        {
9281            return existing.clone();
9282        }
9283        // A seat that changes agent mints its session id from the agent too,
9284        // like a handover: the old agent's uuid is already taken by the CLI.
9285        let fresh = if self.state.seats.contains_key(key) {
9286            handover_seat(key, agent, self.state.next_seat_seed())
9287        } else {
9288            SeatState::new(key, agent, self.state.seed)
9289        };
9290        self.state.seats.insert(key.to_owned(), fresh.clone());
9291        fresh
9292    }
9293
9294    /// The agent now holding seat `key`: `spec`, unless a handover moved the
9295    /// seat to another roster agent, in which case that agent. Nodes that
9296    /// continue a seat's conversation (deliberation, the votes, a reviewer's
9297    /// reconsideration) must keep talking to whoever answered it, not slip
9298    /// back to the agent that failed it.
9299    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9300        match self.state.seats.get(key) {
9301            Some(s) if s.agent != spec.id => {
9302                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9303            }
9304            _ => spec,
9305        }
9306    }
9307
9308    /// A candidate rendered for judging, with the leak policy applied.
9309    fn view(&self, c: &Candidate) -> CandidateView {
9310        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9311            .unwrap_or_default();
9312        let (patch, _) = blind::sanitize_patch(
9313            &format!("candidate {} patch", c.label),
9314            &raw,
9315            &self.state.config.blind,
9316        );
9317        CandidateView {
9318            label: c.label,
9319            branch: c.branch.clone(),
9320            summary: c.summary.clone(),
9321            stat: c.stat.clone(),
9322            patch,
9323        }
9324    }
9325
9326    /// The full candidate set as prompt text, for seats with no live session.
9327    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9328        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9329        prompt::judge(
9330            "(see above)",
9331            &views,
9332            self.roles.judges.len(),
9333            base_short,
9334            "en",
9335        )
9336    }
9337
9338    /// The final-vote prompt with everything a seat that has no session of its
9339    /// own needs: the candidates, the seat's own ranking and reasons, and the
9340    /// anonymised deliberation it took part in (only when there was one, so a
9341    /// handed-over seat never sees more than the seat it replaces did). The
9342    /// `Final vote` heading stays first.
9343    fn vote_prompt_full(
9344        &self,
9345        j: usize,
9346        labels: &[char],
9347        language: &str,
9348        candidates: &[Candidate],
9349        base_short: &str,
9350    ) -> String {
9351        let mut text = format!(
9352            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9353            prompt::final_vote(labels, language),
9354            self.state.instruction,
9355            self.candidate_block(candidates, base_short)
9356        );
9357        if let Some(own) = self
9358            .state
9359            .judgements
9360            .get(j)
9361            .filter(|r| !r.ranking.is_empty())
9362        {
9363            let reasons = own
9364                .reasons
9365                .iter()
9366                .map(|(k, v)| format!("- {k}: {v}"))
9367                .collect::<Vec<_>>()
9368                .join("\n");
9369            text.push_str(&format!(
9370                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9371                own.ranking.iter().collect::<String>(),
9372                if reasons.is_empty() {
9373                    ""
9374                } else {
9375                    ", because:\n"
9376                }
9377            ));
9378        }
9379        if !self.state.deliberation.is_empty() {
9380            text.push_str("\n# What was argued before this vote\n");
9381            for t in self.transcript(&[], j) {
9382                text.push_str(&format!(
9383                    "\n## {}{}\n\n{}\n",
9384                    t.who,
9385                    if t.is_self { " (you)" } else { "" },
9386                    t.body.trim()
9387                ));
9388            }
9389        }
9390        text
9391    }
9392
9393    /// Anonymised transcript for judge `self_idx`.
9394    ///
9395    /// The initial rankings are always the opening statements. Seeding them
9396    /// only when no turn had been taken yet meant every judge after the first
9397    /// argued against a single voice instead of against the actual split — the
9398    /// disagreement is the information, so it is always on the table.
9399    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9400        let mut turns = Vec::new();
9401        for j in &self.state.judgements {
9402            if j.ranking.is_empty() {
9403                continue;
9404            }
9405            let reasons = j
9406                .reasons
9407                .iter()
9408                .map(|(k, v)| format!("- {k}: {v}"))
9409                .collect::<Vec<_>>()
9410                .join("\n");
9411            turns.push(Turn {
9412                who: format!("Judge {} (opening ranking)", j.judge),
9413                is_self: j.judge == self_idx + 1,
9414                body: format!(
9415                    "Ranked {}{}{reasons}",
9416                    j.ranking.iter().collect::<String>(),
9417                    if reasons.is_empty() {
9418                        ""
9419                    } else {
9420                        ", because:\n"
9421                    }
9422                ),
9423            });
9424        }
9425        for t in self
9426            .state
9427            .deliberation
9428            .iter()
9429            .flat_map(|r| r.turns.iter())
9430            .chain(current)
9431        {
9432            turns.push(Turn {
9433                who: format!("Judge {}", t.judge),
9434                is_self: t.judge == self_idx + 1,
9435                body: t.body.clone(),
9436            });
9437        }
9438        turns
9439    }
9440}
9441
9442/// Does this seat still hold the context a follow-up prompt would rely on?
9443fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9444    agent::has_session(spec.kind, seat, sessions)
9445}
9446
9447/// The next entry in `roster` after `start`, never wrapping back to the
9448/// front, whose id is not in `tried` yet.
9449///
9450/// Starts one past `start` rather than at the front of `roster`: `start` is
9451/// the seat's own original position, and a seat whose candidate slot already
9452/// sits on the roster's second entry must fall through to the third next, not
9453/// restart at the first — which is very likely a different candidate's own
9454/// agent already. Never wraps back past `start`, for the same reason: an
9455/// entry earlier in the roster than the seat's own position is almost
9456/// certainly some *other* candidate slot's own agent, and once the tail of
9457/// the roster is exhausted there are no more untried agents for *this* seat
9458/// to fall through to — the caller's fallback chain ends there, exactly as
9459/// "no further untried agents remain in the list for that seat" asks for.
9460///
9461/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
9462/// the same id twice (an operator's `roles.implementers` typo, or a
9463/// `[[agents]]` list reused across roles) must not let
9464/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
9465/// over `roster` either finds an untried id or runs out, so this always
9466/// terminates regardless of duplicates.
9467fn next_untried_in_roster<'a>(
9468    roster: &'a [AgentSpec],
9469    start: usize,
9470    tried: &BTreeSet<String>,
9471) -> Option<&'a AgentSpec> {
9472    roster
9473        .get(start + 1..)?
9474        .iter()
9475        .find(|s| !tried.contains(&s.id))
9476}
9477
9478/// The successor for a seat, shared by all four seat kinds (implement, judge,
9479/// review, advise). `others` holds the ids that *currently* occupy the other
9480/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
9481/// sees them), so roster entries beyond the seat count act as spares: a failed
9482/// seat goes to an agent no other seat holds whenever the roster permits.
9483///
9484/// `carried` is `Some` only for the review loop's carried failure history.
9485/// Order: (1) forward from `start`, never wrapping, untried, not a carried
9486/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
9487/// whole roster: untried and not another seat's occupant; (3) the plain walk
9488/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
9489/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
9490/// a better outcome than an empty seat, and it keeps the answer to "is there
9491/// a successor at all" exactly what it was before occupants were considered,
9492/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
9493/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
9494/// carried failure is a cheaper bet than doubling an agent on the panel, and
9495/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
9496/// the same round are handled one at a time, so a seat later in the batch
9497/// sees an earlier one's new occupant but not yet its own freed agent.
9498fn pick_successor<'a>(
9499    roster: &'a [AgentSpec],
9500    start: usize,
9501    tried: &BTreeSet<String>,
9502    carried: Option<&BTreeSet<String>>,
9503    others: &BTreeSet<String>,
9504) -> Option<&'a AgentSpec> {
9505    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9506    roster
9507        .get(start + 1..)
9508        .and_then(|tail| {
9509            tail.iter()
9510                .filter(free)
9511                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9512        })
9513        .or_else(|| {
9514            carried
9515                .is_some()
9516                .then(|| roster.iter().find(free))
9517                .flatten()
9518        })
9519        .or_else(|| match carried {
9520            Some(c) => next_for_seat(roster, start, tried, c),
9521            None => next_untried_in_roster(roster, start, tried),
9522        })
9523}
9524
9525/// The next agent for a seat that carries its failure history across rounds
9526/// (the review loop). `round_tried` is this round's own bound and starts
9527/// empty every round; `carried_failed` only decides priority.
9528///
9529/// First: an id walking forward from `start`, never wrapping, that is neither
9530/// tried this round nor failed in an earlier one. Only when that is exhausted
9531/// does it rescue: the first roster id (in roster order, so this one *does*
9532/// look before `start`) not yet tried this round, which is by then a carried
9533/// failure. Each id is rescued at most once per round, so it cannot loop.
9534fn next_for_seat<'a>(
9535    roster: &'a [AgentSpec],
9536    start: usize,
9537    round_tried: &BTreeSet<String>,
9538    carried_failed: &BTreeSet<String>,
9539) -> Option<&'a AgentSpec> {
9540    roster
9541        .get(start + 1..)?
9542        .iter()
9543        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9544        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9545}
9546
9547/// Where a reviewer seat starts a round: the agent that last answered it when
9548/// it is still on the roster and not marked failed, else the spec's own agent
9549/// unless it failed, else the next roster agent that has not failed, else the
9550/// spec's own agent again (the whole roster failed). Ids no longer on the
9551/// roster are ignored. An empty roster has no handover, so the spec stands.
9552fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9553    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9554        return spec;
9555    };
9556    let ok = |id: &str| !h.failed.contains(id);
9557    if let Some(last) = h.last_ok.as_deref()
9558        && ok(last)
9559        && let Some(s) = roster.iter().find(|s| s.id == last)
9560    {
9561        return s.clone();
9562    }
9563    if ok(&spec.id) {
9564        return spec;
9565    }
9566    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9567    roster
9568        .get(from + 1..)
9569        .into_iter()
9570        .flatten()
9571        .chain(roster.iter())
9572        .find(|s| ok(&s.id))
9573        .cloned()
9574        .unwrap_or(spec)
9575}
9576
9577/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
9578/// seed so a CLI that mints its session id up front (`--session-id`) never
9579/// reuses the uuid the previous agent already opened under the same seat key.
9580pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9581    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9582}
9583
9584/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
9585/// so the classifier below and the code that builds the message cannot drift.
9586const TIMED_OUT: &str = "timed out";
9587
9588impl FailClass {
9589    /// `None` for an answer; otherwise how the turn failed.
9590    fn of(out: &AgentOutcome) -> Option<Self> {
9591        match out {
9592            AgentOutcome::Ok(_) => None,
9593            AgentOutcome::Quota(_) => Some(Self::Quota),
9594            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9595            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9596            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9597            // Not a failure of the agent. Callers check for it first; `None`
9598            // here means "no class to hand over on".
9599            AgentOutcome::ResourceBlocked(_) => None,
9600        }
9601    }
9602
9603    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
9604    fn stem_word(&self) -> &'static str {
9605        match self {
9606            Self::Quota => "quota",
9607            _ => "handover",
9608        }
9609    }
9610}
9611
9612/// The message's first line with its variable parts removed — digit runs and
9613/// path-like tokens — so "exit code 2" and "exit code 7" read
9614/// as one kind of failure.
9615fn failure_signature(msg: &str) -> String {
9616    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9617    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
9618    // the historical machine key while the process diagnosis uses readable prose.
9619    if let Some(code) = line
9620        .strip_prefix("process exited with no usable output (exit code ")
9621        .and_then(|s| s.strip_suffix(')'))
9622        .and_then(|s| s.parse::<i32>().ok())
9623    {
9624        line = format!("exited with some({code}) and no usable output");
9625    } else if matches!(
9626        line.as_str(),
9627        "process exited with no usable output (no exit code reported)"
9628            | "process exited with no usable output (exit code unavailable)"
9629    ) {
9630        line = "exited with none and no usable output".to_owned();
9631    }
9632    let mut out = Vec::new();
9633    for word in line.split_whitespace() {
9634        if word.contains('/') || word.contains('\\') {
9635            out.push("<path>".to_owned());
9636            continue;
9637        }
9638        let mut w = String::new();
9639        let mut in_digits = false;
9640        for c in word.chars() {
9641            if c.is_ascii_digit() {
9642                if !in_digits {
9643                    w.push('#');
9644                }
9645                in_digits = true;
9646            } else {
9647                in_digits = false;
9648                w.push(c);
9649            }
9650        }
9651        out.push(w);
9652    }
9653    out.join(" ").chars().take(120).collect()
9654}
9655
9656/// Whether a seat that just failed with `cur` may go to the next roster agent.
9657/// A quota or a timeout always may. Any other failure may not when the agent
9658/// before it failed the same way: an error the prompt causes would otherwise
9659/// walk the whole roster. `prev` is the class of the immediately preceding
9660/// agent's failure, so a quota or timeout in between breaks the run of
9661/// identical failures by itself.
9662fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9663    match cur {
9664        FailClass::Quota | FailClass::Timeout => true,
9665        FailClass::Other(_) => prev != Some(cur),
9666    }
9667}
9668
9669/// A short human reason for a failed outcome, for the handover record.
9670fn fail_reason(out: &AgentOutcome) -> String {
9671    match out {
9672        AgentOutcome::Ok(_) => String::new(),
9673        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9674        AgentOutcome::Dropped(o) => format!(
9675            "the CLI dropped the stream ({})",
9676            o.dropped
9677                .as_ref()
9678                .map(|d| d.why.as_str())
9679                .unwrap_or("it ended without delivering its answer")
9680        ),
9681        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9682            e.lines().next().unwrap_or("").chars().take(160).collect()
9683        }
9684    }
9685}
9686
9687/// Note one handover in the run: the structured record and, in the timeline,
9688/// the sentence a person reads. A quota keeps the wording it always had.
9689pub(crate) fn record_handover(
9690    state: &mut RunState,
9691    node: &str,
9692    seat: &str,
9693    from: &str,
9694    to: &str,
9695    class: &FailClass,
9696    reason: &str,
9697) {
9698    let message = if *class == FailClass::Quota {
9699        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9700    } else {
9701        format!("{seat}: handed over {from} -> {to} ({reason})")
9702    };
9703    state.event(node, message);
9704    state.handovers.push(Handover {
9705        at: Timestamp::now(),
9706        node: node.to_owned(),
9707        seat: seat.to_owned(),
9708        from: from.to_owned(),
9709        to: to.to_owned(),
9710        reason: reason.to_owned(),
9711    });
9712}
9713
9714/// Did this reply report running a command whose own CLI never confirmed an
9715/// exit status?
9716///
9717/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9718/// command *finished* (see that type's own doc), so this can only be `true`
9719/// for a command whose completion event carried no readable exit code — not
9720/// for one that simply is not mentioned at all. That is the one signal this
9721/// crate can read, from the same record `state.jobs` renders, about a reply
9722/// standing next to work its own CLI cannot vouch for finishing; it is
9723/// deliberately not a check on the exit code's *value* (a fixer legitimately
9724/// runs a command that fails mid-iteration before it succeeds) and not a
9725/// guess at a command still running in the background (which emits no event
9726/// at all, and so leaves no evidence here to find).
9727fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9728    commands.iter().any(|c| c.exit_code.is_none())
9729}
9730
9731/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9732/// trusted as a verified no-op — the adoption guard's own text-level half.
9733///
9734/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9735/// timed out): a marker only earns the benefit of the doubt from a turn the
9736/// CLI itself vouches for finishing properly, the same house style
9737/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9738/// *fix* report to for `commands`. A candidate that timed out, exited
9739/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9740/// is, whatever prose it wrote — this returns `None` before it ever looks at
9741/// `text`. The remaining guards (the tree really is empty, the evidence is
9742/// non-empty) are the caller's: this only reads what the reply *claimed*.
9743fn verified_noop_claim(
9744    usable: bool,
9745    commands: &[agent::CommandEvidence],
9746    text: &str,
9747) -> Option<String> {
9748    (usable && !has_unconfirmed_command(commands))
9749        .then(|| verdict::verified_noop(text))
9750        .flatten()
9751}
9752
9753fn short(commit: &str) -> String {
9754    commit.chars().take(7).collect()
9755}
9756
9757fn make_executable(path: &Path) -> Result<()> {
9758    #[cfg(unix)]
9759    {
9760        use std::os::unix::fs::PermissionsExt as _;
9761        let mut perms = std::fs::metadata(path)?.permissions();
9762        perms.set_mode(0o755);
9763        std::fs::set_permissions(path, perms)?;
9764    }
9765    #[cfg(not(unix))]
9766    {
9767        let _ = path;
9768    }
9769    Ok(())
9770}
9771
9772/// What every seat in one batch shares: where the answers are attributed, the
9773/// prompt overlay they inherit, and the build cache they are told to use.
9774///
9775/// A struct rather than four more parameters: `wave` also needs the run's
9776/// state (to record who is answering right now) and the attempt number, and
9777/// eight positional arguments is both unreadable and a clippy error.
9778struct WaveCtx<'a> {
9779    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9780    /// paid for it.
9781    run: &'a str,
9782    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9783    node: &'a str,
9784    prompts: &'a Prompts,
9785    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9786    cache: Option<&'a Path>,
9787    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9788    /// `JobRecord::round`. `None` for every other node.
9789    round: Option<usize>,
9790    /// Carry each seat's failed-agent history across waves (the review loop
9791    /// only): start-of-round priority and handover choice read
9792    /// [`RunState::seat_history`], and every answer or failure writes it.
9793    carry_seats: bool,
9794    /// The run's park / interrupt requests, so a wait for a shared resource
9795    /// can be abandoned the moment the operator asks to park.
9796    pauses: Option<(&'a Pause, &'a Pause)>,
9797}
9798
9799/// Shared scheduling policy for operations and spawned seat invocations.
9800fn repository_wait(
9801    state: &RunState,
9802    pauses: Option<(&Pause, &Pause)>,
9803) -> crate::repo_lock::WaitPolicy {
9804    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9805    let inherited = crate::repo_lock::WaitPolicy::current();
9806    crate::repo_lock::WaitPolicy::new(
9807        Duration::from_secs(state.config.graph.timeout_resource_wait),
9808        move || {
9809            inherited.cancelled()
9810                || pauses
9811                    .as_ref()
9812                    .is_some_and(|(a, b)| a.parked() || b.parked())
9813        },
9814    )
9815}
9816
9817fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9818    if error.is::<crate::repo_lock::ResourceBlocked>() {
9819        state.resource_block = Some(crate::run::ResourceBlock {
9820            cause: crate::run::ResourceBlockCause::Other,
9821            node: "repository".into(),
9822            kind: if error.to_string().contains("disk pressure") {
9823                "disk pressure"
9824            } else {
9825                "repository coordination"
9826            }
9827            .into(),
9828            detail: error.to_string(),
9829            at: Timestamp::now(),
9830        });
9831        state.event("resource", error.to_string());
9832    }
9833}
9834
9835async fn coordinate_operation(
9836    state: &mut RunState,
9837    cwd: &Path,
9838    wait: &crate::repo_lock::WaitPolicy,
9839) -> Result<Option<crate::repo_lock::SeatLease>> {
9840    let result =
9841        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9842            .await;
9843    if let Err(e) = &result {
9844        record_repository_block(state, e);
9845    }
9846    result
9847}
9848
9849/// Run one job, honouring the parallelism budget.
9850async fn run_one(
9851    job: SeatJob,
9852    sem: Arc<Semaphore>,
9853    ctx: &WaveCtx<'_>,
9854    state: &mut RunState,
9855    attempt: usize,
9856) -> (SeatState, AgentOutcome) {
9857    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9858        .await
9859        .pop()
9860        .expect("one job in, one result out");
9861    (seat, out)
9862}
9863
9864/// Run every job concurrently, capped by the semaphore, preserving order.
9865///
9866/// Every seat in the batch is recorded into [`RunState::active`] before the
9867/// wave starts and cleared as each answer lands, so the run's own record says
9868/// who is still being waited on rather than only who finished.
9869async fn wave(
9870    jobs: Vec<SeatJob>,
9871    sem: Arc<Semaphore>,
9872    ctx: &WaveCtx<'_>,
9873    state: &mut RunState,
9874    attempt: usize,
9875) -> Vec<(usize, SeatState, AgentOutcome)> {
9876    let WaveCtx {
9877        run,
9878        node,
9879        prompts,
9880        cache,
9881        round,
9882        carry_seats: _,
9883        pauses,
9884    } = *ctx;
9885    for job in &jobs {
9886        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9887    }
9888    if let Err(e) = state.save() {
9889        // A failed persist of "who is answering right now" must not abort the
9890        // wave: the seats are already being asked, and the alternative is
9891        // losing the answers to save a status line nobody may even be
9892        // watching.
9893        tracing::warn!("could not persist in-progress seats: {e:#}");
9894    }
9895    // Hold the shared build cache's lease for the whole batch, not per job:
9896    // several candidates (an implement wave) or a fixer legitimately share
9897    // one cache concurrently within this run, and that stays untouched — a
9898    // single lease taken once for the whole wave and released once it is
9899    // done is what stops a *different* borrower (another run's own wave, its
9900    // e2e/gate, a human's `magi review`) from interleaving a build into the
9901    // same directory while this one is in flight. Best-effort, not
9902    // all-or-nothing: a wave that cannot get the lease within its own
9903    // longest job's budget still runs — an hour of paid implementer calls is
9904    // not thrown away over cache contention — but every write-allowed seat
9905    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9906    // below), the same fallback a read-only seat always gets, rather than
9907    // building into a directory this run was never granted. The identity
9908    // record is still invalidated below either way, so the next tracked
9909    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9910    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9911    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9912    // never borrowed from a job's agent budget (and never charged to it either,
9913    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9914    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9915    let cache_guard = if let Some(cache_dir) = cache {
9916        if jobs_had_a_writer {
9917            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9918            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9919            match acquire_cache_lease(
9920                state,
9921                cache_dir,
9922                &owner,
9923                resource_limit,
9924                node,
9925                node,
9926                &cancel,
9927            )
9928            .await
9929            {
9930                Ok(Some(g)) => Some(g),
9931                // Parking: leave without spawning anything. The run parks at
9932                // the next boundary and the seats are asked afresh on resume.
9933                Ok(None) => {
9934                    if let Some(block) = &mut state.resource_block {
9935                        block.cause = crate::run::ResourceBlockCause::Cancelled {
9936                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9937                        };
9938                    }
9939                    return block_wave(
9940                        jobs,
9941                        state,
9942                        node,
9943                        attempt,
9944                        "parked while waiting for the shared build cache",
9945                    );
9946                }
9947                // An unreadable lease is not contention: keep the old
9948                // fallback (no shared cache for this wave's seats).
9949                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9950                Err(e) => {
9951                    // Writing seats may not build into a cache this run does
9952                    // not hold, and an agent that cannot be given one is not
9953                    // asked at all: the run stops resumably instead.
9954                    let why = format!("{e:#}");
9955                    state.event(
9956                        "resource",
9957                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9958                    );
9959                    return block_wave(jobs, state, node, attempt, &why);
9960                }
9961            }
9962        } else {
9963            None
9964        }
9965    } else {
9966        None
9967    };
9968    // Every job keeps its whole budget: the wait above was not the agent's.
9969    // Restart the seats' clocks so "started" means "being spawned now".
9970    state.seats_spawning(node, attempt);
9971    if let Err(e) = state.save() {
9972        tracing::warn!("could not persist the start of a wave: {e:#}");
9973    }
9974    // Queueing behind this wave's own seats for a parallelism slot is normal
9975    // (each of them is bounded by its own agent timeout); only a wait beyond
9976    // all of them plus the resource limit means the slots never came back.
9977    let slot_limit = resource_limit
9978        + jobs
9979            .iter()
9980            .map(|j| j.timeout)
9981            .max()
9982            .unwrap_or_default()
9983            .saturating_mul(jobs.len() as u32);
9984    let mut set = tokio::task::JoinSet::new();
9985    // Seats announce themselves here the moment they hold a parallelism
9986    // permit, i.e. just before their CLI is spawned.
9987    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9988    let overlay = prompts.overlay(node);
9989    let repository_wait = repository_wait(state, pauses);
9990    for (i, mut job) in jobs.into_iter().enumerate() {
9991        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9992        if cache.is_some() {
9993            job.prompt.push('\n');
9994            job.prompt
9995                .push_str(&prompt::build_cache_note(node, job.allow_write));
9996        }
9997        let sem = Arc::clone(&sem);
9998        let run = run.to_owned();
9999        let node = node.to_owned();
10000        // Only implementers were told about the task's attachments, so only
10001        // their seats get the directory widened for reading.
10002        let attachments = if node == "implement" {
10003            state.attachments.clone()
10004        } else {
10005            Vec::new()
10006        };
10007        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10008        // `prompt::build_cache_note`'s doc for why setting it anyway is
10009        // exactly how a sandboxed reviewer's write refusal got reported as a
10010        // defect in the patch, not a property of its own seat. And a
10011        // write-allowed one is handed it only when the lease above was
10012        // actually acquired: a wave that could not get it (`cache_guard` is
10013        // `None`, see its own comment) must not send seats to build into a
10014        // directory this run does not hold - that is the exact concurrent,
10015        // unmanaged-write race this module exists to prevent, not something
10016        // "proceeding anyway" is allowed to reintroduce.
10017        let cache = cache
10018            .filter(|_| job.allow_write && cache_guard.is_some())
10019            .map(Path::to_path_buf);
10020        let spawned_tx = spawned_tx.clone();
10021        let repository_wait = repository_wait.clone();
10022        let coordination = state.config.coordination.clone();
10023        let fetched_base = state.fetched_base.clone();
10024        set.spawn(async move {
10025            // A wait for a parallelism slot is scheduling, not the agent: it
10026            // is bounded by its own limit and ends as a resource block, never
10027            // as a timeout of the agent that was not started.
10028            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10029                Ok(p) => p,
10030                Err(_) => {
10031                    return (
10032                        i,
10033                        job.seat,
10034                        AgentOutcome::ResourceBlocked(format!(
10035                            "no parallel slot became free within {}s",
10036                            slot_limit.as_secs()
10037                        )),
10038                        false,
10039                    );
10040                }
10041            };
10042            spawned_tx.send(job.seat.key.clone()).ok();
10043            let mut seat = job.seat;
10044            let out = repository_wait
10045                .scope(agent::invoke(
10046                    &job.spec,
10047                    &mut seat,
10048                    &Invocation {
10049                        coordination: coordination.as_ref(),
10050                        fetched_base: fetched_base.as_ref(),
10051                        cwd: &job.cwd,
10052                        prompt: &job.prompt,
10053                        timeout: job.timeout,
10054                        allow_write: job.allow_write,
10055                        sessions: job.sessions,
10056                        artifacts: &job.artifacts,
10057                        stem: &job.stem,
10058                        run: &run,
10059                        node: &node,
10060                        cache_dir: cache.as_deref(),
10061                        attachments: &attachments,
10062                        writable: &[],
10063                    },
10064                ))
10065                .await;
10066            let interrupted = out
10067                .as_ref()
10068                .err()
10069                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10070            let out = match out {
10071                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10072                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10073                // Billed work the CLI failed to hand over is not an ordinary
10074                // failure, but its text is the CLI's raw error JSON, not an
10075                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10076                // read it as one by forgetting to check. `usable()` is always
10077                // false here (dropped implies an empty response), so this has
10078                // to be checked before the catch-all `Failed` below or the
10079                // one shape this exists for is lost with the rest.
10080                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10081                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10082                Ok(o) => AgentOutcome::Failed(format!(
10083                    "process exited with no usable output ({})",
10084                    crate::notices::exit_status(o.exit_code)
10085                )),
10086                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10087                    AgentOutcome::ResourceBlocked(e.to_string())
10088                }
10089                Err(e) => AgentOutcome::Failed(e.to_string()),
10090            };
10091            (i, seat, out, interrupted)
10092        });
10093    }
10094    drop(spawned_tx);
10095    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10096    loop {
10097        let joined = tokio::select! {
10098            Some(key) = spawned_rx.recv() => {
10099                // The clock of this seat starts now, not when it queued.
10100                state.seat_spawned(&key);
10101                if let Err(e) = state.save() {
10102                    tracing::warn!("could not persist a seat's start: {e:#}");
10103                }
10104                continue;
10105            }
10106            joined = set.join_next() => joined,
10107        };
10108        let Some(joined) = joined else {
10109            break;
10110        };
10111        let (i, seat, out, interrupted) = match joined {
10112            Ok(v) => v,
10113            // No seat to clear: a panicked task never reported which one it
10114            // was. The defensive sweep below this loop is what stops that
10115            // seat's `active` entry from surviving forever.
10116            Err(e) => {
10117                tracing::error!("agent task panicked: {e}");
10118                continue;
10119            }
10120        };
10121        state.seat_finished(&seat.key);
10122        if interrupted
10123            && node == "implement"
10124            && let Some(label) = state
10125                .candidates
10126                .iter()
10127                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10128                .map(|candidate| candidate.label)
10129        {
10130            state.candidate_preservation.insert(
10131                label,
10132                "interrupted implementation awaiting persistence".into(),
10133            );
10134            state.interrupted_implementations.insert(label);
10135        }
10136        if let AgentOutcome::ResourceBlocked(why) = &out
10137            && state.resource_block.is_none()
10138        {
10139            state.resource_block = Some(crate::run::ResourceBlock {
10140                cause: crate::run::ResourceBlockCause::Other,
10141                node: node.to_owned(),
10142                kind: if why.contains("disk pressure") {
10143                    "disk pressure"
10144                } else if why.contains("repository coordination") {
10145                    "repository coordination"
10146                } else {
10147                    "parallel slot"
10148                }
10149                .to_owned(),
10150                detail: why.clone(),
10151                at: Timestamp::now(),
10152            });
10153            let activity = if interrupted {
10154                "agent execution interrupted"
10155            } else {
10156                "no agent was started"
10157            };
10158            state.event("resource", format!("{node}: {activity} - {why}"));
10159        }
10160        record_jobs(state, node, round, &seat.key, &out);
10161        if let Err(e) = state.save() {
10162            tracing::warn!("could not persist a seat's completion: {e:#}");
10163        }
10164        if collected.len() <= i {
10165            collected.resize_with(i + 1, || None);
10166        }
10167        collected[i] = Some((i, seat, out));
10168    }
10169    // Belt-and-braces for the panic branch above: every seat this exact batch
10170    // started shares this `(node, attempt)` pair, and every seat that finished
10171    // normally already cleared itself, so anything left tagged with it here
10172    // can only be a panicked task's leftover. Cleared unconditionally rather
10173    // than left to read as still answering forever.
10174    if state
10175        .active
10176        .values()
10177        .any(|a| a.node == node && a.attempt == attempt)
10178    {
10179        state
10180            .active
10181            .retain(|_, a| !(a.node == node && a.attempt == attempt));
10182        if let Err(e) = state.save() {
10183            tracing::warn!("could not persist the end of a wave: {e:#}");
10184        }
10185    }
10186    // Whether or not the lease above was actually held, several worktrees
10187    // may just have built into the cache with nothing here able to name one
10188    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
10189    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
10190    // clean it might not have strictly needed; trusting a stale match would
10191    // cost it a wrong answer.
10192    if let Some(cache_dir) = cache
10193        && jobs_had_a_writer
10194    {
10195        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10196    }
10197    if let Some(guard) = cache_guard {
10198        guard.release();
10199    }
10200    collected.into_iter().flatten().collect()
10201}
10202
10203/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
10204/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
10205/// this is data collection, not the fix-specific completion contract in
10206/// [`Runner::continue_fix_report`], and applies regardless of which node
10207/// asked.
10208///
10209/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
10210/// evidence from; `Failed` does not, and correctly contributes nothing — a
10211/// timeout or crash is not itself evidence about a command the seat may have
10212/// started.
10213fn record_jobs(
10214    state: &mut RunState,
10215    node: &str,
10216    round: Option<usize>,
10217    seat: &str,
10218    out: &AgentOutcome,
10219) {
10220    let commands: &[agent::CommandEvidence] = match out {
10221        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10222        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10223    };
10224    let checked_at = Timestamp::now();
10225    for c in commands {
10226        state.jobs.push(JobRecord {
10227            node: node.to_owned(),
10228            round,
10229            seat: seat.to_owned(),
10230            id: c.id.clone(),
10231            description: c.description.clone(),
10232            checked_at,
10233            status: match c.exit_code {
10234                Some(0) => JobStatus::Completed,
10235                Some(_) => JobStatus::Failed,
10236                None => JobStatus::Unknown,
10237            },
10238            exit_code: c.exit_code,
10239            result_summary: c.result_summary.clone(),
10240            source: c.source.clone(),
10241        });
10242    }
10243}
10244
10245/// Is a review round clean, given how many reviewer seats answered against
10246/// how many the round expected?
10247///
10248/// A seat that never answered (timeout, crash, unparsable output) is not a
10249/// seat that read the patch and found nothing — treating it as such is
10250/// exactly the bug this function exists to close. Under the default `block`
10251/// policy a missing seat can never be clean; `warn` still requires the seats
10252/// that *did* answer to have found nothing blocking and verification to be
10253/// green.
10254///
10255/// `quota_missing` narrows that `block` default for exactly one cause of
10256/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
10257/// a session limit lifts by the very next round buys nothing — the seat is
10258/// asked again with the same quota — so once every missing seat is accounted
10259/// for by a quota loss (and at least one seat *did* answer, so a decision has
10260/// something to rest on) the round is decided on the panel that could answer,
10261/// same as `warn` would. A panel that lost every seat to quota is not
10262/// decided here: `answered == 0` falls through to the existing `block`
10263/// fallback so a fully collapsed panel still waits rather than landing on no
10264/// review at all.
10265fn round_is_clean(
10266    blocking: usize,
10267    e2e_ok: bool,
10268    answered: usize,
10269    expected: usize,
10270    quota_missing: usize,
10271    policy: IncompleteReviewPolicy,
10272) -> bool {
10273    if blocking != 0 || !e2e_ok {
10274        return false;
10275    }
10276    if answered == expected || policy == IncompleteReviewPolicy::Warn {
10277        return true;
10278    }
10279    answered > 0 && expected - answered <= quota_missing
10280}
10281
10282/// The review loop's own conclusion, derived entirely from its persisted
10283/// round records and the round budget that produced them — never from
10284/// `status`, so a reentry (or `gate`/`merge` reading it independently)
10285/// recomputes the identical answer regardless of what an earlier node in the
10286/// same walk, or a previous walk, did to `status`.
10287///
10288/// `None` while more rounds remain to try, including when review never ran
10289/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
10290/// gone clean, or the budget is spent, or the tree has stopped moving (see
10291/// [`STAGNANT_LIMIT`]), the answer is one of two things:
10292///
10293/// - An incomplete panel that raised nothing is missing input, not a
10294///   verified tree — never a hand-off candidate, whatever verification said
10295///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
10296/// - Otherwise, green e2e on the last round hands off (see
10297///   [`Runner::stop_reviewing`]); red e2e blocks.
10298///
10299/// A last round whose own verification is still `ResourceBlocked` — magi
10300/// itself never got a command to run, not evidence the patch is broken —
10301/// is neither: this returns `None` for it too, the same as "more rounds
10302/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
10303/// handling of that shape) instead of this cheap recomputation guessing a
10304/// verdict a real attempt never produced.
10305pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10306    if max_rounds == 0
10307        || reviews.last().is_some_and(|r| {
10308            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10309        })
10310    {
10311        return Some(RunStatus::Gating);
10312    }
10313    let last = reviews.last()?;
10314    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10315    if reviews.len() < max_rounds && !stagnant {
10316        return None;
10317    }
10318    if last.incomplete() && last.blocking == 0 {
10319        return Some(RunStatus::Blocked);
10320    }
10321    if last.e2e_status() == E2eStatus::ResourceBlocked
10322        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10323    {
10324        return None;
10325    }
10326    Some(if last.e2e.iter().all(CommandOutcome::ok) {
10327        RunStatus::Gating
10328    } else {
10329        RunStatus::Blocked
10330    })
10331}
10332
10333/// How long a re-ask may take, given the budget the first attempt had.
10334///
10335/// A `nudged` retry is a request to restate an answer the seat has already
10336/// worked out: it carries no new work, so it does not deserve the original
10337/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
10338/// 133 seconds while a third sat for over ten minutes on a resumed session
10339/// holding 410 KB of prior output - and because the retry had inherited the
10340/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
10341/// judging round whose other seats were long finished.
10342///
10343/// A quarter of the budget, with a floor so that a deliberately short timeout
10344/// does not collapse to nothing. A retry that re-sends the whole prompt
10345/// (because the seat kept no context) is the original job again, and keeps the
10346/// original budget.
10347pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10348    if nudged {
10349        (full / 4).max(Duration::from_secs(120)).min(full)
10350    } else {
10351        full
10352    }
10353}
10354
10355/// Run a wave and parse each reply, re-asking the seats whose reply was
10356/// unusable.
10357///
10358/// The re-ask is a nudge rather than the whole prompt again when the seat still
10359/// holds its conversation, which is the difference between a cheap retry and
10360/// paying for the entire candidate set twice.
10361///
10362/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
10363/// successor in `roster` is handed to it instead of being re-asked: the
10364/// handover is the retry. A seat with no successor left (a single-agent
10365/// roster, the roster's tail) is nudged as before, up to `retries` times. So
10366/// the asks to one seat in one node number at most
10367/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
10368/// is asked once (a dropped stream is nudged first), and only the last agent
10369/// of the chain gets the `retries` same-agent nudges. Each roster agent is
10370/// tried at most once per seat, walking forward from the seat's own position and never wrapping
10371/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
10372/// other failure stops the chain when the previous agent failed the same way
10373/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
10374/// [`has_context`] is false and the job's own full prompt and full budget are
10375/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
10376/// intermediate ones are not losses) and is returned as a failure like any
10377/// other absent seat — the caller decides whether the panel still has a
10378/// quorum. An empty `roster` disables handover: failures are nudged as they
10379/// always were, and a quota is simply lost. A reply that fails to parse or
10380/// validate is the prompt's doing and is only ever nudged, never handed over.
10381///
10382/// The returned [`SeatState`] names the agent that answered (or tried last).
10383#[allow(clippy::too_many_arguments)]
10384async fn ask_json_wave<T>(
10385    jobs: Vec<SeatJob>,
10386    sem: Arc<Semaphore>,
10387    retries: usize,
10388    roster: &[AgentSpec],
10389    ctx: &WaveCtx<'_>,
10390    losses: &mut Vec<QuotaLoss>,
10391    state: &mut RunState,
10392    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10393) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10394where
10395    T: serde::de::DeserializeOwned + Send + 'static,
10396{
10397    ask_wave_with(
10398        jobs,
10399        sem,
10400        retries,
10401        roster,
10402        ctx,
10403        losses,
10404        state,
10405        &|text: &str| {
10406            let v = verdict::extract_json::<T>(text)?;
10407            validate(&v)?;
10408            Ok(v)
10409        },
10410    )
10411    .await
10412}
10413
10414/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
10415/// a node whose answer is prose (deliberation) shares the same handover,
10416/// failure classification, quota bookkeeping and bounds instead of a copy.
10417///
10418/// A seat handed to another roster agent is sent the job's `handover` prompt
10419/// (when it has one) rather than `prompt`: the new agent has no session, so a
10420/// resume-style prompt would be incomplete. That holds for the handover ask
10421/// and for every nudge to that agent whose `has_context` is false.
10422#[allow(clippy::too_many_arguments)]
10423async fn ask_wave_with<T>(
10424    jobs: Vec<SeatJob>,
10425    sem: Arc<Semaphore>,
10426    retries: usize,
10427    roster: &[AgentSpec],
10428    ctx: &WaveCtx<'_>,
10429    losses: &mut Vec<QuotaLoss>,
10430    state: &mut RunState,
10431    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10432) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10433where
10434    T: Send + 'static,
10435{
10436    let n = jobs.len();
10437    let originals: Vec<SeatJob> = jobs;
10438    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10439    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10440    // Nudges each seat's *current* agent has taken — 0 for a first-ask
10441    // answer, N once it has gone through N nudges. Read back once this
10442    // returns, so a caller building a history record (`ReviewRecord`) can
10443    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
10444    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
10445    // field's own doc.
10446    let mut nudges: Vec<usize> = vec![0; n];
10447    // The agent now occupying each seat, the ids it has already been through,
10448    // where in the roster the walk began, the class of the last failure, and
10449    // the stem word of a handover not yet asked (full prompt, full budget).
10450    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10451    let mut tried: Vec<BTreeSet<String>> = specs
10452        .iter()
10453        .map(|s| BTreeSet::from([s.id.clone()]))
10454        .collect();
10455    let starts: Vec<usize> = specs
10456        .iter()
10457        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10458        .collect();
10459    let carry = ctx.carry_seats && !roster.is_empty();
10460    // Carried across rounds: ids that failed the seat earlier, and how the
10461    // last failure went (so a repeat of it is not handed over again).
10462    let carried: Vec<BTreeSet<String>> = originals
10463        .iter()
10464        .map(|j| {
10465            state
10466                .seat_history
10467                .get(&j.seat.key)
10468                .filter(|_| carry)
10469                .map(|h| h.failed.clone())
10470                .unwrap_or_default()
10471        })
10472        .collect();
10473    let mut prev: Vec<Option<FailClass>> = originals
10474        .iter()
10475        .map(|j| {
10476            state
10477                .seat_history
10478                .get(&j.seat.key)
10479                .filter(|_| carry)
10480                .and_then(|h| h.last_fail.clone())
10481        })
10482        .collect();
10483    let next_agent =
10484        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10485            let others: BTreeSet<String> = specs
10486                .iter()
10487                .enumerate()
10488                .filter(|(j, _)| *j != i)
10489                .map(|(_, s)| s.id.clone())
10490                .collect();
10491            pick_successor(
10492                roster,
10493                starts[i],
10494                tried,
10495                carry.then(|| &carried[i]),
10496                &others,
10497            )
10498            .cloned()
10499        };
10500    let mut fresh: Vec<Option<String>> = vec![None; n];
10501    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
10502    let mut pending: Vec<usize> = (0..n).collect();
10503
10504    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
10505    // asks: an agent with a successor is asked once and handed over, and only
10506    // a seat with no successor spends `retries` nudges on the same agent. This
10507    // only guarantees the loop's own termination whatever those say.
10508    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10509    for round in 0..max_rounds {
10510        if pending.is_empty() {
10511            break;
10512        }
10513        let mut batch = Vec::with_capacity(pending.len());
10514        let mut renudged: Vec<&str> = Vec::new();
10515        for &i in &pending {
10516            let src = &originals[i];
10517            // A seat now held by another agent than the job named has no
10518            // session of its own: it gets the full-context prompt whenever
10519            // it is asked in full (the handover ask, a nudge it cannot
10520            // resume).
10521            let full: &str = match &src.handover {
10522                Some(h) if specs[i].id != src.spec.id => h,
10523                _ => &src.prompt,
10524            };
10525            // The prompt and the budget are one decision: a nudge restates
10526            // finished work, a re-sent prompt redoes it.
10527            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10528                (
10529                    full.to_owned(),
10530                    src.timeout,
10531                    format!("{}-{word}-{}", src.stem, specs[i].id),
10532                )
10533            } else if nudges[i] == 0 {
10534                (full.to_owned(), src.timeout, src.stem.clone())
10535            } else {
10536                renudged.push(src.seat.key.as_str());
10537                let why = done[i]
10538                    .as_ref()
10539                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
10540                    .unwrap_or_else(|| "no parsable answer".to_owned());
10541                let nudge = prompt::nudge(&why);
10542                let nudged = has_context(&specs[i], &seats[i], src.sessions);
10543                let prompt = if nudged {
10544                    nudge
10545                } else {
10546                    format!("{full}\n\n---\n\n{nudge}")
10547                };
10548                (
10549                    prompt,
10550                    retry_budget(src.timeout, nudged),
10551                    format!("{}-retry{}", src.stem, nudges[i]),
10552                )
10553            };
10554            batch.push(SeatJob {
10555                spec: specs[i].clone(),
10556                seat: seats[i].clone(),
10557                cwd: src.cwd.clone(),
10558                prompt,
10559                timeout,
10560                allow_write: src.allow_write,
10561                sessions: src.sessions,
10562                artifacts: src.artifacts.clone(),
10563                stem,
10564                handover: None,
10565            });
10566        }
10567
10568        if !renudged.is_empty() {
10569            state.event(
10570                ctx.node,
10571                format!("retry {round}: re-asking {}", renudged.join(", ")),
10572            );
10573        }
10574        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10575        let mut still = Vec::new();
10576        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10577            seats[i] = seat;
10578            // No agent was asked: nothing to hand over, nudge, remember or
10579            // charge. The caller sees `resource_block` and stops the node.
10580            if let AgentOutcome::ResourceBlocked(why) = &out {
10581                done[i] = Some(Err(anyhow::anyhow!("{why}")));
10582                continue;
10583            }
10584            let class = FailClass::of(&out);
10585            // A dropped stream is nudged first (the conversation is still
10586            // there to pick up); only a seat whose nudges are spent hands over.
10587            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10588                && nudges[i] < retries
10589                && !roster.is_empty();
10590            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10591                let next = should_hand_over(prev[i].as_ref(), &cur)
10592                    .then(|| next_agent(i, &tried[i], &specs))
10593                    .flatten();
10594                if carry {
10595                    let h = state
10596                        .seat_history
10597                        .entry(originals[i].seat.key.clone())
10598                        .or_default();
10599                    h.failed.insert(specs[i].id.clone());
10600                    h.last_fail = Some(cur.clone());
10601                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10602                        h.last_ok = None;
10603                    }
10604                    // Saved before the next agent is asked, so a restart in
10605                    // between does not forget who failed.
10606                    if let Err(e) = state.save() {
10607                        tracing::warn!("could not persist a seat's failure history: {e:#}");
10608                    }
10609                }
10610                if let Some(next) = next {
10611                    record_handover(
10612                        state,
10613                        ctx.node,
10614                        &originals[i].seat.key,
10615                        &specs[i].id,
10616                        &next.id,
10617                        &cur,
10618                        &fail_reason(&out),
10619                    );
10620                    tried[i].insert(next.id.clone());
10621                    prev[i] = Some(cur.clone());
10622                    seats[i] =
10623                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10624                    specs[i] = next;
10625                    fresh[i] = Some(cur.stem_word().to_owned());
10626                    nudges[i] = 0;
10627                    done[i] = Some(Err(anyhow::anyhow!(
10628                        "handed over after: {}",
10629                        fail_reason(&out)
10630                    )));
10631                    still.push(i);
10632                    continue;
10633                }
10634            }
10635            if carry
10636                && !nudge_first
10637                && let Some(cur) = class.clone()
10638            {
10639                // The chain ended here (no successor, or a repeated failure).
10640                let h = state
10641                    .seat_history
10642                    .entry(originals[i].seat.key.clone())
10643                    .or_default();
10644                h.failed.insert(specs[i].id.clone());
10645                h.last_fail = Some(cur);
10646            }
10647            let parsed = match out {
10648                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10649                AgentOutcome::Quota(o) => {
10650                    last_quota[i] = o.quota.clone();
10651                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10652                }
10653                // Not a parseable answer: the nudge loop re-asks it, which is
10654                // exactly what a dropped stream needs. Just don't hand its raw
10655                // error JSON to `extract_json`.
10656                AgentOutcome::Dropped(o) => {
10657                    let why = o
10658                        .dropped
10659                        .as_ref()
10660                        .map(|d| d.why.as_str())
10661                        .unwrap_or("the CLI ended the stream without delivering its answer");
10662                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10663                }
10664                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10665                    Err(anyhow::anyhow!(e))
10666                }
10667            };
10668            let quota = class == Some(FailClass::Quota);
10669            let failed = parsed.is_err();
10670            if !failed && matches!(ctx.node, "judge" | "vote") {
10671                // A validated answer restores this seat's quorum presence,
10672                // including on the normal path after an all-seat block.
10673                state
10674                    .quota
10675                    .retain(|loss| loss.seat != originals[i].seat.key);
10676            }
10677            done[i] = Some(parsed);
10678            if carry && !failed {
10679                let h = state
10680                    .seat_history
10681                    .entry(originals[i].seat.key.clone())
10682                    .or_default();
10683                h.failed.remove(&specs[i].id);
10684                h.last_ok = Some(specs[i].id.clone());
10685                h.last_fail = None;
10686            }
10687            // Do not re-ask a rate-limited seat (quota) — a retry is known to
10688            // fail the same way; and never re-ask a seat that already parsed.
10689            // A failed agent that still has a successor is not re-asked
10690            // either: the handover was its remedy and has just been refused
10691            // (the chain stops on a repeated failure class). A seat with no
10692            // successor left (a single-agent roster, the roster's tail, or an
10693            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10694            let agent_failure = class.is_some()
10695                && !nudge_first
10696                && !roster.is_empty()
10697                && next_agent(i, &tried[i], &specs).is_some();
10698            if failed && !quota && !agent_failure && nudges[i] < retries {
10699                nudges[i] += 1;
10700                still.push(i);
10701            }
10702        }
10703        pending = still;
10704    }
10705
10706    // One loss per seat whose chain ended on a quota: the intermediate ones
10707    // were absorbed by a handover and are not losses.
10708    for (i, q) in last_quota.into_iter().enumerate() {
10709        if let Some(quota) = q {
10710            losses.push(QuotaLoss::from_report(
10711                originals[i].seat.key.clone(),
10712                ctx.node,
10713                Some(&quota),
10714            ));
10715        }
10716    }
10717
10718    // A stalled quorum already has its own recovery checkpoint. Refresh
10719    // its missing seats before deciding whether it can resume.
10720    if state.status != RunStatus::Stalled
10721        && ctx.node != "deliberate"
10722        && n > 0
10723        && losses.len() == n
10724        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
10725    {
10726        block_machine_outage(state, ctx.node, losses);
10727        state.quota.append(losses);
10728    }
10729
10730    seats
10731        .into_iter()
10732        .zip(done)
10733        .zip(nudges)
10734        .map(|((seat, res), attempts)| {
10735            (
10736                seat,
10737                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10738                attempts,
10739            )
10740        })
10741        .collect()
10742}
10743
10744/// A node with no available agent has no task verdict. Preserve the node
10745/// checkpoint and record why a subsequent explicit resume is needed.
10746pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
10747    let mut auth: Vec<_> = losses
10748        .iter()
10749        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
10750        .map(|loss| loss.agent.clone())
10751        .collect();
10752    auth.sort();
10753    auth.dedup();
10754    let (kind, detail) = if auth.is_empty() {
10755        (
10756            "agent usage limit",
10757            "All seats reached their agent usage limits; resume after usage is available"
10758                .to_owned(),
10759        )
10760    } else {
10761        (
10762            "agent authentication",
10763            format!(
10764                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
10765                auth.join(", ")
10766            ),
10767        )
10768    };
10769    state.resource_block = Some(crate::run::ResourceBlock {
10770        node: node.to_owned(),
10771        kind: kind.to_owned(),
10772        cause: crate::run::ResourceBlockCause::Other,
10773        detail: detail.clone(),
10774        at: Timestamp::now(),
10775    });
10776    state.event(node, detail);
10777}
10778
10779/// Did the wave end without asking any agent (see [`block_wave`])?
10780fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10781    results
10782        .iter()
10783        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10784}
10785
10786/// Settle a wave that never spawned anything: every job answers
10787/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10788/// else about the seat (session, history, quota) is touched.
10789fn block_wave(
10790    jobs: Vec<SeatJob>,
10791    state: &mut RunState,
10792    node: &str,
10793    attempt: usize,
10794    why: &str,
10795) -> Vec<(usize, SeatState, AgentOutcome)> {
10796    state.resource_wait = None;
10797    state
10798        .active
10799        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10800    if let Err(e) = state.save() {
10801        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10802    }
10803    jobs.into_iter()
10804        .enumerate()
10805        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10806        .collect()
10807}
10808
10809/// Acquire the shared build cache's lease, waiting out contention within
10810/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10811/// unbounded wait is never acceptable).
10812///
10813/// A first, non-blocking check happens before ever waiting; if it finds the
10814/// lease busy, that fact is logged as a `verify` event *and* flushed with
10815/// [`RunState::save`] immediately — not only once the wait finally succeeds
10816/// or gives up — so a `magi show` run by a different process while this one
10817/// is still waiting reads a `run.json` that says so, rather than whatever it
10818/// looked like before the wait started. The same applies to the terminal
10819/// failure: logged and saved before this returns `Err`, so a caller that
10820/// could not get the lease at all still leaves a legible record of why.
10821async fn acquire_cache_lease(
10822    state: &mut RunState,
10823    cache_dir: &Path,
10824    owner: &crate::cache::Owner,
10825    budget: Duration,
10826    node: &str,
10827    context: &str,
10828    cancel: &(dyn Fn() -> bool + Sync),
10829) -> Result<Option<crate::cache::Guard>> {
10830    let home = crate::run::home();
10831    let started = Instant::now();
10832    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10833        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10834        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10835        Err(e) => {
10836            state.event(
10837                "verify",
10838                format!("{context}: could not check the shared build cache: {e:#}"),
10839            );
10840            if let Err(e2) = state.save() {
10841                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10842            }
10843            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10844        }
10845    };
10846    state.event(
10847        "verify",
10848        format!(
10849            "{context}: waiting for the shared build cache at {} ({})",
10850            cache_dir.display(),
10851            busy.describe()
10852        ),
10853    );
10854    // Seats of this node are waiting, not running: say so, so the countdown a
10855    // reader sees only ever measures an agent that has been spawned.
10856    state.resource_wait = Some(crate::run::ResourceWait {
10857        node: context.to_owned(),
10858        kind: "shared build cache".to_owned(),
10859        holder: busy.describe(),
10860        since: Timestamp::now(),
10861        limit_secs: budget.as_secs(),
10862    });
10863    if let Err(e) = state.save() {
10864        tracing::warn!("could not persist a cache wait: {e:#}");
10865    }
10866    let remaining = budget.saturating_sub(started.elapsed());
10867    let waited = crate::cache::wait_for_unless(
10868        &home,
10869        cache_dir,
10870        owner,
10871        remaining,
10872        Duration::from_secs(5),
10873        cancel,
10874    )
10875    .await;
10876    state.resource_wait = None;
10877    let cause = match &waited {
10878        Ok(Some(_)) => None,
10879        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10880        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10881            Some(crate::run::ResourceBlockCause::Timeout {
10882                waited_secs: started.elapsed().as_secs(),
10883                limit_secs: budget.as_secs(),
10884            })
10885        }
10886        Err(_) => Some(crate::run::ResourceBlockCause::Other),
10887    };
10888    if let Some(cause) = cause {
10889        let detail = match &waited {
10890            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10891            Err(error) => format!("{error:#}"),
10892        };
10893        state.resource_block = Some(crate::run::ResourceBlock {
10894            cause,
10895            node: node.to_owned(),
10896            kind: "shared build cache".into(),
10897            detail,
10898            at: Timestamp::now(),
10899        });
10900        state.event(
10901            "resource",
10902            state.resource_block.as_ref().unwrap().description(),
10903        );
10904    }
10905    if let Err(error) = state.save() {
10906        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10907    }
10908    waited
10909}
10910
10911/// The lease state could not be read at all (unreadable lease file, a
10912/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10913/// known to be contended, so a wave degrades to running without a shared cache
10914/// instead of stopping on a hold that [`crate::triage`] could never release.
10915#[derive(Debug)]
10916struct CacheCheckError(String);
10917
10918impl std::fmt::Display for CacheCheckError {
10919    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10920        f.write_str(&self.0)
10921    }
10922}
10923
10924impl std::error::Error for CacheCheckError {}
10925
10926/// Keep a specific acquisition outcome when the caller adds command evidence.
10927fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10928    state
10929        .resource_block
10930        .get_or_insert_with(|| crate::run::ResourceBlock {
10931            cause: crate::run::ResourceBlockCause::Other,
10932            node: node.to_owned(),
10933            kind: "shared build cache".into(),
10934            detail: format!("{error:#}"),
10935            at: Timestamp::now(),
10936        });
10937    if let Err(error) = state.save() {
10938        tracing::warn!("could not persist cache resource block: {error:#}");
10939    }
10940}
10941
10942/// Run `body` — a verify command batch — while holding the shared build
10943/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10944/// never interleave with another borrower's build against the same
10945/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10946/// timeout, or a human's own `magi review`. See the `cache` module doc for
10947/// why this matters more than Cargo's own per-target locking covers — two
10948/// *different* worktrees building the same package name/version into one
10949/// cache directory is a staleness bug, not a lock contention one.
10950///
10951/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10952/// agent waves do. Repository scheduling also has its own bounded,
10953/// cancellable resource-wait policy. Contention must not consume the
10954/// verification budget: `body` receives the full `budget` after both leases
10955/// are acquired and the cache is fresh. Without a shared cache, only
10956/// repository coordination is acquired before handing over the full budget.
10957///
10958/// A lease that cannot be acquired within the resource wait is reported as a single
10959/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10960/// verification — the same shape a spawn failure already takes in
10961/// [`run_commands`], so a caller need not special-case it.
10962#[allow(clippy::too_many_arguments)]
10963async fn with_cache_lease<'s, F, Fut>(
10964    state: &'s mut RunState,
10965    cache_dir: Option<&Path>,
10966    node: &str,
10967    seat: &str,
10968    worktree: &Path,
10969    head: &str,
10970    budget: Duration,
10971    context: &str,
10972    wait: &crate::repo_lock::WaitPolicy,
10973    body: F,
10974) -> Result<(Vec<CommandOutcome>, bool)>
10975where
10976    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10977    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10978{
10979    // Match editing waves: cache first, repository second. In particular,
10980    // pre_gate must never hold the repository while waiting for a wave's cache.
10981    let started = Instant::now();
10982    let home = crate::run::home();
10983    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10984    let guard = if let Some(cache_dir) = cache_dir {
10985        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10986        match acquire_cache_lease(
10987            state,
10988            cache_dir,
10989            &owner,
10990            resource_limit,
10991            node,
10992            context,
10993            &|| wait.cancelled(),
10994        )
10995        .await
10996        .and_then(|g| {
10997            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10998        }) {
10999            Ok(g) => Some(g),
11000            Err(e) => {
11001                record_cache_error(state, node, &e);
11002                return Ok((
11003                    vec![CommandOutcome {
11004                        command: "(waiting for the shared build cache)".into(),
11005                        code: None,
11006                        output_tail: e.to_string(),
11007                        duration_ms: started.elapsed().as_millis() as u64,
11008                        resource_blocked: true,
11009                    }],
11010                    false,
11011                ));
11012            }
11013        }
11014    } else {
11015        None
11016    };
11017    let _coordination = coordinate_operation(state, worktree, wait).await?;
11018    if let Some(cache_dir) = cache_dir {
11019        let identity = crate::cache::Identity::new(worktree, head);
11020        if let Err(e) = crate::cache::ensure_fresh(
11021            &home,
11022            cache_dir,
11023            &identity,
11024            &state.config.verify.cargo_manifests(),
11025        ) {
11026            record_cache_error(state, node, &e);
11027            state.event(
11028                "verify",
11029                format!(
11030                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11031                    worktree.display(),
11032                    short(head)
11033                ),
11034            );
11035            if let Some(guard) = guard {
11036                guard.release();
11037            }
11038            return Ok((
11039                vec![CommandOutcome {
11040                    command: "(confirming the shared build cache is fresh)".into(),
11041                    code: None,
11042                    output_tail: e.to_string(),
11043                    duration_ms: started.elapsed().as_millis() as u64,
11044                    resource_blocked: true,
11045                }],
11046                false,
11047            ));
11048        }
11049    }
11050    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11051    if !timed_out_pids.is_empty() {
11052        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11053    }
11054    // Drop repository coordination before releasing the cache, reversing the
11055    // acquisition order. A cancelled/error future also drops both guards.
11056    drop(_coordination);
11057    if let Some(guard) = guard {
11058        guard.release();
11059    }
11060    Ok((outcomes, retried))
11061}
11062
11063/// Poll `pids` — commands [`run_commands`] reports as still running when its
11064/// own timeout elapsed — until every one is confirmed gone, or
11065/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11066///
11067/// Real confirmation where confirmation is possible, not a substitute for
11068/// full process-tree observation: a grandchild the timed-out process spawned
11069/// and that survives independently of it is invisible to a pid check the
11070/// same way it always was, and continuing to observe and collect *that*
11071/// stays a different piece of work with its own owner. This only narrows a
11072/// fixed blind wait into an actual check of the pids this process does know
11073/// about.
11074async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11075    wait_for_pids_with(
11076        pids,
11077        crate::proc::pid_alive,
11078        LEASE_RELEASE_POLL,
11079        LEASE_RELEASE_MAX_WAIT,
11080    )
11081    .await;
11082}
11083
11084/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
11085/// interval and ceiling supplied by the caller, so the polling *logic* -
11086/// returns as soon as every pid reports dead, gives up at the ceiling
11087/// otherwise - is testable on millisecond durations without asking the real
11088/// OS about a pid at all.
11089async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11090    pids: &[u32],
11091    alive: F,
11092    poll: Duration,
11093    max_wait: Duration,
11094) {
11095    let deadline = Instant::now() + max_wait;
11096    loop {
11097        if pids.iter().all(|&pid| !alive(pid)) {
11098            return;
11099        }
11100        if Instant::now() >= deadline {
11101            return;
11102        }
11103        tokio::time::sleep(poll).await;
11104    }
11105}
11106
11107/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
11108/// admission that it could not even get a verify command to run, as opposed
11109/// to evidence the command actually produced? A caller that would otherwise
11110/// read a resource-blocked outcome as a red command must check this first:
11111/// see [`Runner::gate`], which retries rather than records `Blocked` when
11112/// this is true.
11113fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11114    outcomes.iter().any(|o| o.resource_blocked)
11115}
11116
11117/// What [`Runner::gate_fix_round`] decided.
11118pub(crate) enum GateFix {
11119    /// The tree changed and `verify.e2e` is still green: run the gate again.
11120    Retry,
11121    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
11122    /// last failure stands and the run ends blocked.
11123    Stop,
11124    /// `verify.e2e` could not run after the fix (magi's own contention):
11125    /// decide nothing now, a later reentry retries.
11126    Defer,
11127}
11128
11129/// Is every red command in `outcomes` an ordinary failure the code could
11130/// explain: it ran, exited non-zero, and said something?
11131///
11132/// A timeout, a spawn failure and a killed process all leave `code` `None`;
11133/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
11134/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
11135/// what the command is stays the gate's business.
11136fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11137    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11138    red.peek().is_some()
11139        && red.all(|o| {
11140            !o.resource_blocked
11141                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11142                && !o.output_tail.trim().is_empty()
11143        })
11144}
11145
11146/// Describe one verify command's outcome for the event log, distinguishing a
11147/// build/link failure — the toolchain never produced a binary to run — from
11148/// an actual test failure, since only the latter is a verdict on the patch.
11149fn e2e_outcome_label(o: &CommandOutcome) -> String {
11150    if o.ok() {
11151        return "pass".to_owned();
11152    }
11153    let reason = if o.build_failed() {
11154        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11155    } else {
11156        format!("FAIL ({:?})", o.code)
11157    };
11158    format!(
11159        "{reason}\n{}",
11160        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11161    )
11162}
11163
11164/// Run `verify.e2e`, retrying once if the first attempt could not build or
11165/// link — a build/link failure is frequently a race against a shared
11166/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
11167/// `verify` event per command, tagged with `context` (normally `"round N"`)
11168/// so the two call sites that need this — the ordinary per-round leg in
11169/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
11170/// it will ever call a round green — read identically in the event log.
11171async fn run_e2e_with_retry(
11172    state: &mut RunState,
11173    shell: &[String],
11174    commands: &[String],
11175    worktree: &Path,
11176    timeout: Duration,
11177    context: &str,
11178) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11179    let (mut e2e, mut timed_out_pids) = run_commands(
11180        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11181    )
11182    .await;
11183    for o in &e2e {
11184        state.event(
11185            "verify",
11186            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11187        );
11188    }
11189    // A build/link failure is not a verdict on the patch — it is frequently a
11190    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
11191    // one retry before letting a red like that decide the round.
11192    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11193    if verify_retried {
11194        state.event(
11195            "verify",
11196            format!(
11197                "{context}: verify could not build/link, not a test result — retrying once \
11198                 before concluding"
11199            ),
11200        );
11201        let retried = run_commands(
11202            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11203        )
11204        .await;
11205        e2e = retried.0;
11206        // Both attempts' timeouts matter, not just the last one: the first
11207        // attempt's descendants may still be alive alongside the retry's.
11208        timed_out_pids.extend(retried.1);
11209        for o in &e2e {
11210            state.event(
11211                "verify",
11212                format!(
11213                    "{context}: retry `{}` -> {}",
11214                    o.command,
11215                    e2e_outcome_label(o)
11216                ),
11217            );
11218        }
11219    }
11220    (e2e, verify_retried, timed_out_pids)
11221}
11222
11223/// Run configured shell commands in `cwd`, in order. The second element is
11224/// the pid of every command that hit `timeout` and was still running when
11225/// this stopped waiting on it (best-effort: `None` when the platform did not
11226/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
11227/// that releases a shared resource afterward needs to know.
11228///
11229/// Records `task` into [`RunState::active`] at every command boundary
11230/// (`RunState::task_command`) and clears it once the whole list has run
11231/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
11232/// for minutes with no seat and no output of its own to show for it (see
11233/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
11234/// apart from "not yet run" without this), and this is the only place that
11235/// knows which command is running right now and how many are left. Three
11236/// saves per command — start, not per second — matching the same "only at a
11237/// boundary" rule [`wave`] already follows for seats.
11238#[allow(clippy::too_many_arguments)]
11239async fn run_commands(
11240    state: &mut RunState,
11241    node: &str,
11242    task: &str,
11243    attempt: usize,
11244    shell: &[String],
11245    commands: &[String],
11246    cwd: &Path,
11247    timeout: Duration,
11248) -> (Vec<CommandOutcome>, Vec<u32>) {
11249    if commands.is_empty() {
11250        // Nothing to mark as running and nothing to clear — an empty list
11251        // means "not configured", and touching `active` (or the disk) over
11252        // that would be a write for every round of a repo with no
11253        // `verify.e2e` / `verify.gate` commands at all.
11254        return (Vec::new(), Vec::new());
11255    }
11256    let mut out = Vec::new();
11257    let mut timed_out_pids = Vec::new();
11258    let total = commands.len();
11259    for (idx, command) in commands.iter().enumerate() {
11260        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11261        if let Err(e) = state.save() {
11262            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11263        }
11264        let started = Instant::now();
11265        let mut cmd = tokio::process::Command::new(&shell[0]);
11266        cmd.quiet();
11267        // Verify the checkout as a standalone caller. Worker ownership and
11268        // pinned executable selection belong to this driver, not to tests or
11269        // tools it runs. Preserve home/seat context and Running's containment.
11270        for key in crate::worker::INVOCATION_ENV {
11271            cmd.env_remove(key);
11272        }
11273        cmd.args(&shell[1..])
11274            .arg(command)
11275            .current_dir(cwd)
11276            .stdin(std::process::Stdio::null())
11277            .stdout(std::process::Stdio::piped())
11278            .stderr(std::process::Stdio::piped())
11279            .kill_on_drop(true);
11280        let mut outputs = Vec::new();
11281        let protected = (|| -> Result<()> {
11282            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
11283                outputs.push(guard);
11284            }
11285            if let Some(cache) = state.config.cache_dir() {
11286                outputs.push(crate::retention::OutputLease::cache(
11287                    &crate::run::home(),
11288                    &cache,
11289                    false,
11290                )?);
11291            }
11292            Ok(())
11293        })();
11294        if let Err(e) = protected {
11295            out.push(CommandOutcome {
11296                command: command.clone(),
11297                code: None,
11298                output_tail: format!("output ownership unavailable: {e:#}"),
11299                duration_ms: 0,
11300                resource_blocked: true,
11301            });
11302            continue;
11303        }
11304        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
11305        let (code, body) = match spawned {
11306            Ok(mut running) => {
11307                let pid = running.child().id();
11308                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
11309                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
11310                let mut paths = vec![cwd.to_owned()];
11311                if let Some(cache) = state.config.cache_dir() {
11312                    paths.push(cache);
11313                }
11314                let mut pressure = None;
11315                let status = tokio::select! {
11316                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
11317                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
11318                        pressure = Some(reason); None
11319                    }
11320                };
11321                let (code, failure) = match status {
11322                    Some(Ok(Ok(status))) => (
11323                        status.code(),
11324                        status
11325                            .code()
11326                            .is_none()
11327                            .then(|| format!("command terminated: {status}")),
11328                    ),
11329                    Some(Ok(Err(error))) => {
11330                        (None, Some(format!("failed waiting for command: {error}")))
11331                    }
11332                    Some(Err(_)) => {
11333                        // Leased cleanup confirms process-tree death itself.
11334                        // Unleased commands still need the caller's PID check
11335                        // after kill-on-drop, before releasing cache ownership.
11336                        if !running.coordinated()
11337                            && let Some(pid) = pid
11338                        {
11339                            timed_out_pids.push(pid);
11340                        }
11341                        (
11342                            None,
11343                            Some(format!("timed out after {}s", timeout.as_secs())),
11344                        )
11345                    }
11346                    None => (None, None),
11347                };
11348                running.finish().await;
11349                if let Some(reason) = pressure {
11350                    state.resource_block = Some(crate::run::ResourceBlock {
11351                        cause: crate::run::ResourceBlockCause::Other,
11352                        node: node.to_owned(),
11353                        kind: "disk pressure".to_owned(),
11354                        detail: reason.clone(),
11355                        at: Timestamp::now(),
11356                    });
11357                    out.push(CommandOutcome {
11358                        command: command.clone(),
11359                        code: None,
11360                        output_tail: reason,
11361                        duration_ms: started.elapsed().as_millis() as u64,
11362                        resource_blocked: true,
11363                    });
11364                    break;
11365                }
11366                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11367                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11368                if let Some(failure) = failure {
11369                    body.push('\n');
11370                    body.push_str(&failure);
11371                }
11372                (code, body)
11373            }
11374            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11375        };
11376        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
11377        if !body.is_empty() {
11378            // The full text is kept whatever the summary dropped. A fresh
11379            // name per command so a resume or retry never overwrites an
11380            // earlier run's output. A failed write changes no verdict.
11381            match crate::run::write_verification_artifact(state, task, &body) {
11382                Ok(name) if summary != body => {
11383                    summary = format!("[full output: artifacts/{name}]\n{summary}");
11384                }
11385                Ok(_) => {}
11386                Err(e) => {
11387                    let warning = format!("could not save the full {task} output: {e:#}");
11388                    tracing::warn!("{warning}");
11389                    state.event(node, warning);
11390                    if summary != body {
11391                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
11392                    }
11393                }
11394            }
11395        }
11396        out.push(CommandOutcome {
11397            command: command.clone(),
11398            code,
11399            output_tail: summary,
11400            duration_ms: started.elapsed().as_millis() as u64,
11401            resource_blocked: false,
11402        });
11403    }
11404    state.task_finished(task);
11405    if let Err(e) = state.save() {
11406        tracing::warn!("could not persist the end of {task}: {e:#}");
11407    }
11408    (out, timed_out_pids)
11409}
11410
11411/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
11412/// section (`report::run`) and in the `merge` event this node records — for
11413/// the operator to run by hand.
11414///
11415/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
11416/// branch whose ruleset forbids merge commits (GitHub's "must not contain
11417/// merge commits", or "require linear history") rejects the push a `--no-ff`
11418/// merge would produce, which is exactly the guidance this function replaces.
11419/// `message`'s first line becomes the squash commit's subject, matching the
11420/// note `report::run` prints alongside this command — see that function for
11421/// why an explicit subject is not optional there.
11422fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11423    let repo = repo.display();
11424    match style {
11425        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11426        MergeStyle::Squash => {
11427            // The subject sits inside double quotes, and a title an agent
11428            // wrote may carry the characters that break out of them.
11429            let subject = message
11430                .lines()
11431                .next()
11432                .unwrap_or(branch)
11433                .replace(['\\', '"', '$', '`'], "");
11434            format!(
11435                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11436            )
11437        }
11438        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11439    }
11440}
11441
11442/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
11443/// under the hood, rejects a `title` over 256 characters and the whole
11444/// command fails — no PR at all, for a run whose body was otherwise fine
11445/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
11446/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
11447/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
11448/// this bound. It is a margin, not a guarantee — a title packed
11449/// with multi-unit characters could still in principle land close to the
11450/// edge, but a real task title's occasional emoji or accented letter fits
11451/// comfortably inside it.
11452const PR_TITLE_MAX: usize = 240;
11453
11454/// A pull request title from the opening line of `text`, or `None` when that
11455/// line is not English (GitHub text is) or has no letters.
11456///
11457/// A line within `max` is kept as is. A longer one is cut at the end of its
11458/// first sentence when that falls inside `max`, else at a word boundary with a
11459/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
11460/// make every merely-truncated title look non-English). The language check
11461/// runs on the kept text before any mark is added, so only what GitHub will
11462/// show is judged: an English opening followed by non-ASCII far past the cut
11463/// still passes.
11464fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11465    let line = queue::first_line(text)?;
11466    let chars: Vec<char> = line.chars().collect();
11467    let (kept, mark) = if chars.len() <= max {
11468        (line.to_owned(), "")
11469    } else if let Some(end) = sentence_end(&chars, max) {
11470        (chars[..end].iter().collect::<String>(), "")
11471    } else {
11472        let room = max.saturating_sub(3);
11473        // Cut at the last space inside the room; when the char just past the
11474        // room is a space the room already ends on a word.
11475        let cut = if chars[room].is_whitespace() {
11476            room
11477        } else {
11478            chars[..room]
11479                .iter()
11480                .rposition(|c| c.is_whitespace())
11481                .unwrap_or(room)
11482        };
11483        let head: String = chars[..cut].iter().collect();
11484        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11485        (head.to_owned(), "...")
11486    };
11487    if kept.is_empty() {
11488        return None;
11489    }
11490    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11491        return None;
11492    }
11493    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11494        return None;
11495    }
11496    Some(format!("{kept}{mark}"))
11497}
11498
11499/// The char length of the first sentence of `chars` when it ends within `max`
11500/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
11501/// abbreviations so `e.g. foo` does not end a title early.
11502fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11503    const MIN: usize = 20;
11504    for i in MIN..max.min(chars.len()) {
11505        if !matches!(chars[i], '.' | '!' | '?') {
11506            continue;
11507        }
11508        let Some(&next) = chars.get(i + 1) else {
11509            continue;
11510        };
11511        if !next.is_whitespace() {
11512            continue;
11513        }
11514        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11515        if after.is_some_and(|c| c.is_ascii_lowercase()) {
11516            continue;
11517        }
11518        let word: String = chars[..i]
11519            .iter()
11520            .rev()
11521            .take_while(|c| !c.is_whitespace())
11522            .collect::<Vec<_>>()
11523            .into_iter()
11524            .rev()
11525            .collect();
11526        let word = word.to_ascii_lowercase();
11527        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11528            continue;
11529        }
11530        let end = chars[..i]
11531            .iter()
11532            .rposition(|c| !c.is_whitespace())
11533            .map_or(i, |p| p + 1);
11534        return Some(end);
11535    }
11536    None
11537}
11538
11539/// What `merge = "pr"` (and the merge commit of the other modes) says about a
11540/// change: a title and a body describing what was *implemented*, not the task
11541/// that asked for it. A task reads as a request; a reader of the merged
11542/// history wants the change.
11543struct PrMessage {
11544    title: String,
11545    body: String,
11546}
11547
11548impl PrMessage {
11549    /// Title, blank line, body. The first line is the squash/merge commit
11550    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
11551    /// has to stay one sensible line.
11552    fn commit_message(&self) -> String {
11553        format!("{}\n\n{}", self.title, self.body)
11554    }
11555}
11556
11557/// The text after a leading `TITLE:` (any case) on `line`.
11558fn title_marker(line: &str) -> Option<&str> {
11559    let line = line.trim();
11560    let head = line.get(..6)?;
11561    head.eq_ignore_ascii_case("title:")
11562        .then(|| line[6..].trim())
11563}
11564
11565/// The implementer's own one-line title: the `TITLE:` line the implement
11566/// prompt asks for at the top of its SUMMARY. Candidate commits are all
11567/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
11568/// source, and a title that says as much is refused here too.
11569fn summary_title(summary: &str) -> Option<String> {
11570    let first = summary.lines().find(|l| !l.trim().is_empty())?;
11571    let raw = title_marker(first)?;
11572    if raw.is_empty() {
11573        return None;
11574    }
11575    let title = queue::title_from(raw, PR_TITLE_MAX);
11576    let lower = title.to_ascii_lowercase();
11577    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11578        return None;
11579    }
11580    Some(title)
11581}
11582
11583/// How `open_review`'s instruction begins; see [`landing_title`].
11584const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11585
11586/// Marker `open_review` gives a candidate that nothing in the roster wrote.
11587pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11588
11589/// Does this run review work that already existed, rather than implement a
11590/// task? Runs recorded before `reviewed_commits` existed carry only the
11591/// candidate marker.
11592pub(crate) fn is_review_run(state: &RunState) -> bool {
11593    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11594}
11595
11596/// The title of a review-only run: the subject of the oldest commit under
11597/// review. Later commits are usually fixups, and `instruction` is the review
11598/// prompt, which says nothing about the change. GitHub text is English, so a
11599/// non-ASCII or blank subject yields `None` and the caller's neutral title.
11600fn review_title(state: &RunState) -> Option<String> {
11601    english_subject(
11602        state.reviewed_commits.as_ref()?.first()?,
11603        state.config.graph.github_english(),
11604    )
11605}
11606
11607/// `raw` as a pull request title, or `None` when it is blank, not English
11608/// (GitHub text is), or one of magi's own candidate commit subjects.
11609fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11610    let raw = raw.trim();
11611    if raw.is_empty() {
11612        return None;
11613    }
11614    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11615        return None;
11616    }
11617    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11618        return None;
11619    }
11620    let title = queue::title_from(raw, PR_TITLE_MAX);
11621    let lower = title.to_ascii_lowercase();
11622    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11623        return None;
11624    }
11625    Some(title)
11626}
11627
11628/// What a review-only run's branch says about itself, read at the moment the
11629/// pull request is opened.
11630#[derive(Debug, Clone, PartialEq, Eq)]
11631struct BranchFacts {
11632    /// `(subject, body)` of each commit, oldest first.
11633    commits: Vec<(String, String)>,
11634    /// Trimmed `git diff --stat`.
11635    stat: String,
11636}
11637
11638/// Longest diff stat shown: this many file lines plus the summary line.
11639const STAT_FILE_LINES: usize = 25;
11640/// Cap on the commit list, well inside GitHub's 65536-character body limit.
11641const COMMITS_MAX_CHARS: usize = 20_000;
11642
11643/// Read the commits and diff stat of `base..branch`. `None` when git cannot
11644/// say or finds nothing, so the caller falls back to what the run recorded.
11645async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11646    let commits = git::commit_log(repo, base, branch).await.ok()?;
11647    if commits.is_empty() {
11648        return None;
11649    }
11650    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11651    let lines: Vec<&str> = stat.lines().collect();
11652    let stat = if lines.len() > STAT_FILE_LINES + 1 {
11653        let omitted = lines.len() - 1 - STAT_FILE_LINES;
11654        let more = format!(" ... {omitted} more file(s)");
11655        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11656        kept.push(&more);
11657        kept.push(lines[lines.len() - 1]);
11658        kept.join("\n")
11659    } else {
11660        lines.join("\n")
11661    };
11662    Some(BranchFacts { commits, stat })
11663}
11664
11665/// Defang what would break the surrounding markdown: a closing `</details>`
11666/// and a code fence.
11667fn markdown_safe(text: &str) -> String {
11668    text.replace("</details>", "&lt;/details&gt;")
11669        .replace("\x60\x60\x60", "~~~")
11670}
11671
11672fn neutral_title(state: &RunState, winner: char) -> String {
11673    format!(
11674        "chore: land candidate {} of run {}",
11675        winner.to_ascii_uppercase(),
11676        state.id
11677    )
11678}
11679
11680/// Reuse the graph's bounded fixer without starting another graph or driver.
11681/// Copy the working state back even on errors so spent rounds remain visible.
11682pub(crate) async fn repair_landing_gate(
11683    state: &mut RunState,
11684    winner: &Candidate,
11685    outcomes: &[CommandOutcome],
11686) -> Result<GateFix> {
11687    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11688        return Ok(GateFix::Stop);
11689    }
11690    let mut runner = Runner {
11691        roles: state.config.resolve_roles()?,
11692        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11693        state: state.clone(),
11694        pause: Pause::new(),
11695        interrupt: Pause::new(),
11696    };
11697    let result = runner.gate_fix_round(winner, outcomes).await;
11698    *state = runner.state;
11699    result
11700}
11701
11702/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
11703/// A completed failure is terminal on that SHA. Resource waits remain retryable.
11704pub(crate) async fn verify_current_head(
11705    state: &mut RunState,
11706    winner: &Candidate,
11707    wait: &crate::repo_lock::WaitPolicy,
11708) -> Result<bool> {
11709    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11710    // Preserve the previous record before replacing it, including on crash recovery.
11711    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
11712        state
11713            .verification_history
11714            .push(crate::run::VerificationEvidence {
11715                superseded_by: Some(head.clone()),
11716                archived_at: Some(Timestamp::now()),
11717                fixes_spent: Some(state.gate_fixes.len()),
11718                gate_head: None,
11719                gate_ran: false,
11720                gate: Vec::new(),
11721                e2e: state.head_e2e.take(),
11722            });
11723    }
11724    if let Some(e) = &state.head_e2e
11725        && e.head == head
11726    {
11727        if e.passed {
11728            return Ok(true);
11729        }
11730        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
11731            state.status = RunStatus::Blocked;
11732            state.save()?;
11733            return Ok(false);
11734        }
11735    } else if state.reviews.last().is_some_and(|r| {
11736        r.verified_head.as_deref() == Some(head.as_str())
11737            && (r.e2e_status() == E2eStatus::Passed
11738                || (r.e2e_status() == E2eStatus::NotConfigured
11739                    && state.config.verify.e2e.is_empty()))
11740    }) || state
11741        .gate_repair_e2e
11742        .as_ref()
11743        .is_some_and(|e| e.head == head && e.passed)
11744    {
11745        return Ok(true);
11746    }
11747    if let Some(e) = &state.gate_repair_e2e
11748        && e.head == head
11749        && !e.passed
11750        && !e.outcomes.is_empty()
11751        && !verify_inconclusive(&e.outcomes)
11752    {
11753        state.status = RunStatus::Blocked;
11754        state.save()?;
11755        return Ok(false);
11756    }
11757    state.head_e2e = Some(crate::run::GateRepairE2e {
11758        head: head.clone(),
11759        passed: false,
11760        outcomes: Vec::new(),
11761    });
11762    state.save()?;
11763    let commands = state.config.verify.e2e.clone();
11764    let shell = state.config.shell();
11765    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11766    let cache_dir = state.config.cache_dir();
11767    let outcomes = if commands.is_empty() {
11768        Vec::new()
11769    } else {
11770        let (outcomes, _) = with_cache_lease(
11771            state,
11772            cache_dir.as_deref(),
11773            "e2e",
11774            "e2e",
11775            &winner.worktree,
11776            &head,
11777            timeout,
11778            "current head e2e",
11779            wait,
11780            |state, budget| async move {
11781                run_e2e_with_retry(
11782                    state,
11783                    &shell,
11784                    &commands,
11785                    &winner.worktree,
11786                    budget,
11787                    "current head e2e",
11788                )
11789                .await
11790            },
11791        )
11792        .await?;
11793        outcomes
11794    };
11795    let unresolved = verify_inconclusive(&outcomes);
11796    let passed = !unresolved
11797        && outcomes.iter().all(CommandOutcome::ok)
11798        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11799    state.head_e2e = Some(crate::run::GateRepairE2e {
11800        head: head.clone(),
11801        passed,
11802        outcomes,
11803    });
11804    if !passed && !unresolved {
11805        state.status = RunStatus::Blocked;
11806        state.event(
11807            "e2e",
11808            format!("verify.e2e failed on head {head}; nothing published"),
11809        );
11810    }
11811    state.save()?;
11812    Ok(passed)
11813}
11814
11815/// Recheck a repair's e2e requirement across failures, resource deferrals and
11816/// process interruptions. A successful result only applies to its exact SHA.
11817pub(crate) async fn check_gate_repair_e2e(
11818    state: &mut RunState,
11819    winner: &Candidate,
11820    wait: &crate::repo_lock::WaitPolicy,
11821) -> Result<GateFix> {
11822    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11823        return Ok(GateFix::Retry);
11824    }
11825    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11826    if state
11827        .gate_repair_e2e
11828        .as_ref()
11829        .is_some_and(|r| r.head == head && r.passed)
11830    {
11831        return Ok(GateFix::Retry);
11832    }
11833    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11834        head: head.clone(),
11835        passed: false,
11836        outcomes: Vec::new(),
11837    });
11838    state.save()?;
11839    let commands = state.config.verify.e2e.clone();
11840    let outcomes = if commands.is_empty() {
11841        Vec::new()
11842    } else {
11843        let shell = state.config.shell();
11844        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11845        let cache_dir = state.config.cache_dir();
11846        let (outcomes, _) = with_cache_lease(
11847            state,
11848            cache_dir.as_deref(),
11849            "e2e",
11850            "e2e",
11851            &winner.worktree,
11852            &head,
11853            timeout,
11854            "gate repair e2e",
11855            wait,
11856            |state, budget| async move {
11857                run_e2e_with_retry(
11858                    state,
11859                    &shell,
11860                    &commands,
11861                    &winner.worktree,
11862                    budget,
11863                    "gate repair e2e",
11864                )
11865                .await
11866            },
11867        )
11868        .await?;
11869        outcomes
11870    };
11871    let deferred = verify_inconclusive(&outcomes);
11872    let passed = outcomes.iter().all(CommandOutcome::ok)
11873        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11874    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11875        head,
11876        passed,
11877        outcomes,
11878    });
11879    state.save()?;
11880    if deferred {
11881        Ok(GateFix::Defer)
11882    } else if !passed {
11883        state.event(
11884            "gate",
11885            "verify.e2e failed after gate repair; nothing published",
11886        );
11887        Ok(GateFix::Stop)
11888    } else {
11889        Ok(GateFix::Retry)
11890    }
11891}
11892
11893/// Run the same configured gate for a landing head and a fresh graph.
11894pub(crate) async fn run_landing_gate(
11895    state: &mut RunState,
11896    winner: &Candidate,
11897    wait: &crate::repo_lock::WaitPolicy,
11898) -> Result<Vec<CommandOutcome>> {
11899    let shell = state.config.shell();
11900    let gate_commands = state.config.verify.gate.clone();
11901    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11902    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11903        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
11904        Vec::new()
11905    } else {
11906        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11907        let cache_dir = state.config.cache_dir();
11908        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11909        let (outcomes, _) = with_cache_lease(
11910            state,
11911            cache_dir.as_deref(),
11912            "gate",
11913            "gate",
11914            &winner.worktree,
11915            &head,
11916            timeout,
11917            "final gate",
11918            wait,
11919            |state, budget| {
11920                let shell = shell.clone();
11921                let gate_commands = gate_commands.clone();
11922                let winner = winner.clone();
11923                async move {
11924                    Runner::run_pre_gate(state, &winner).await;
11925                    // Bind command evidence after mechanical fixes, before commands start.
11926                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
11927                    let (outcomes, timed_out_pids) = run_commands(
11928                        state,
11929                        "gate",
11930                        "gate",
11931                        0,
11932                        &shell,
11933                        &gate_commands,
11934                        &winner.worktree,
11935                        budget,
11936                    )
11937                    .await;
11938                    (outcomes, false, timed_out_pids)
11939                }
11940            },
11941        )
11942        .await?;
11943        outcomes
11944    };
11945    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
11946    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
11947        let mut outcomes = outcomes;
11948        outcomes.push(CommandOutcome {
11949            command: "gate head stability".into(), code: None,
11950            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
11951            duration_ms: 0, resource_blocked: false,
11952        });
11953        return Ok(outcomes);
11954    }
11955    if outcomes.is_empty() {
11956        // Nothing configured to check — distinct from every other
11957        // silence in this run's event log, since an empty `gate` alone
11958        // no longer says whether the gate ran at all (see
11959        // `RunState::gate_ran`'s own doc).
11960        state.event("gate", "no gate commands configured; not tested");
11961    }
11962    for o in &outcomes {
11963        state.event(
11964            "gate",
11965            format!(
11966                "`{}` -> {}",
11967                o.command,
11968                if o.ok() {
11969                    "pass".to_owned()
11970                } else {
11971                    format!(
11972                        "FAIL ({:?})\n{}",
11973                        o.code,
11974                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11975                    )
11976                }
11977            ),
11978        );
11979    }
11980    Ok(outcomes)
11981}
11982
11983/// The pull request title to hand to `land::merge_subject`. A review-only run
11984/// opened by an earlier build titled its pull request with the review prompt;
11985/// that title is dropped (empty, so the fallback applies) rather than landed.
11986/// Any other title, including an operator's rename, passes through untouched,
11987/// and so does every title of a run that implements a task.
11988pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11989    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11990        ""
11991    } else {
11992        pr_title
11993    }
11994}
11995
11996/// What the squash subject falls back to when the pull request title is empty
11997/// or candidate-shaped: for a review-only run the derived title, never the
11998/// review prompt held in `instruction`.
11999pub fn landing_subject_source(state: &RunState) -> String {
12000    if is_review_run(state) {
12001        let winner = state.candidates.first().map_or('A', |c| c.label);
12002        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12003    }
12004    state.instruction.clone()
12005}
12006
12007/// `summary` without its `TITLE:` line, which the pull request title already
12008/// carries.
12009fn summary_without_title(summary: &str) -> String {
12010    let mut lines = summary.trim().lines().peekable();
12011    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12012        lines.next();
12013    }
12014    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12015}
12016
12017/// The pull request title and body for the winning candidate.
12018///
12019/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12020/// the task's own opening line via [`queue::title_from`] when there is none.
12021/// `state.instruction` can open with blank lines (`task_text` only rejects a
12022/// body that is blank *entirely*), which `title_from` skips.
12023///
12024/// Publishing uses an independent description generated from the final diff
12025/// and structured evidence. Internal summaries and reports remain local.
12026/// Non-publishing previews may display the original task and candidate summary.
12027/// The footer retains the run and candidate provenance.
12028#[cfg(test)]
12029fn pr_message(state: &RunState, winner: char) -> PrMessage {
12030    pr_message_with(state, winner, None)
12031}
12032
12033/// [`pr_message`] with what the branch of a review-only run says about itself.
12034/// `facts` is ignored for a run that implements a task.
12035#[cfg(test)]
12036fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12037    let raw = pr_message_raw(
12038        state,
12039        winner,
12040        facts,
12041        crate::pr_outcome::FinalHead::Unreported,
12042        ChangeNote::NotAsked,
12043    );
12044    let id = crate::scrub::Identity::current();
12045    PrMessage {
12046        title: crate::scrub::scrub(&raw.title, &id),
12047        body: crate::scrub::scrub(&raw.body, &id),
12048    }
12049}
12050
12051/// What the public description may say about the change itself.
12052#[derive(Debug, Clone, Copy)]
12053enum ChangeNote<'a> {
12054    /// A local preview or non-publishing merge mode: no public generation needed.
12055    NotAsked,
12056    /// A summary written from the final diff.
12057    Summary(&'a crate::run::ChangeSummary),
12058    /// A summary was asked for and could not be produced; the text says why.
12059    Unavailable(&'a str),
12060}
12061
12062fn pr_message_raw(
12063    state: &RunState,
12064    winner: char,
12065    facts: Option<&BranchFacts>,
12066    head: crate::pr_outcome::FinalHead<'_>,
12067    change: ChangeNote<'_>,
12068) -> PrMessage {
12069    if !matches!(change, ChangeNote::NotAsked) {
12070        let (title, mut body) = match change {
12071            ChangeNote::Summary(c) => (
12072                c.title.clone(),
12073                format!("## Summary\n\n{}\n\n", c.body.trim()),
12074            ),
12075            ChangeNote::Unavailable(why) => (
12076                neutral_title(state, winner),
12077                format!(
12078                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12079                ),
12080            ),
12081            ChangeNote::NotAsked => unreachable!(),
12082        };
12083        body.push_str(&public_pr_evidence(state, head));
12084        body.push_str(&format!(
12085            "\n---\nmagi:run/{} magi:candidate-{}\n",
12086            state.id,
12087            winner.to_ascii_lowercase()
12088        ));
12089        return PrMessage { title, body };
12090    }
12091    pr_preview(state, winner, facts, head)
12092}
12093
12094/// Append public facts from the run records after every prose generation,
12095/// including posting-gate rewrites. The agent cannot drop this provenance.
12096fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12097    let mut body = crate::pr_outcome::review_rounds(state);
12098    body.push_str(&crate::pr_outcome::final_verification(state, head));
12099    let open = state.last_round_findings();
12100    if !open.is_empty() {
12101        body.push_str("## Open review findings\n\n");
12102        for f in open {
12103            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12104        }
12105        body.push('\n');
12106    }
12107    if let Some(c) = state.change_summary.as_ref()
12108        && head == crate::pr_outcome::FinalHead::At(&c.head)
12109    {
12110        body.push_str(&format!(
12111            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12112            c.base, c.head
12113        ));
12114    }
12115    body
12116}
12117
12118/// Local-only preview of the operator's task and candidate summary.
12119fn pr_preview(
12120    state: &RunState,
12121    winner: char,
12122    facts: Option<&BranchFacts>,
12123    head: crate::pr_outcome::FinalHead<'_>,
12124) -> PrMessage {
12125    let summary = state
12126        .candidates
12127        .iter()
12128        .find(|c| c.label == winner)
12129        .map(|c| c.summary.as_str())
12130        .unwrap_or_default();
12131    // The fallback is the operator's own words and may not be English; GitHub
12132    // text always is, so a non-English task gets a neutral title instead.
12133    let review = is_review_run(state);
12134    let title = if review {
12135        facts
12136            .and_then(|f| {
12137                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12138            })
12139            .or_else(|| review_title(state))
12140            .or_else(|| {
12141                state
12142                    .candidates
12143                    .iter()
12144                    .find(|c| c.label == winner)
12145                    .filter(|c| !c.branch.starts_with("magi/"))
12146                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12147            })
12148            .unwrap_or_else(|| neutral_title(state, winner))
12149    } else {
12150        summary_title(summary).unwrap_or_else(|| {
12151            english_title(
12152                &state.instruction,
12153                PR_TITLE_MAX,
12154                state.config.graph.github_english(),
12155            )
12156            .unwrap_or_else(|| neutral_title(state, winner))
12157        })
12158    };
12159
12160    let mut body = String::new();
12161    let what = summary_without_title(summary);
12162    if !what.is_empty() {
12163        body.push_str("## Summary\n\n");
12164        body.push_str(&what);
12165        body.push_str("\n\n");
12166    }
12167
12168    // Review history uses structured facts across all rounds, never reports.
12169    body.push_str(&crate::pr_outcome::review_rounds(state));
12170    body.push_str(&crate::pr_outcome::final_verification(state, head));
12171    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12172
12173    let open = state.open_findings();
12174    if !open.is_empty() {
12175        body.push_str("## Open review findings\n\n");
12176        for f in &open {
12177            body.push_str(&format!(
12178                "- `{}` [{:?}] {}\n",
12179                f.id, f.severity, "Context retained in local review records."
12180            ));
12181        }
12182        body.push('\n');
12183    }
12184
12185    if let Some(fix) = fix
12186        && !fix.rejected.is_empty()
12187    {
12188        body.push_str("## Declined by the fixer\n\n");
12189        for r in &fix.rejected {
12190            body.push_str(&format!(
12191                "- `{}`: {}\n",
12192                r.id, "Reason retained in local review records."
12193            ));
12194        }
12195        body.push('\n');
12196    }
12197
12198    if review {
12199        // The review prompt is not the task; list what the branch carries.
12200        body.push_str("## Commits under review\n\n");
12201        if let Some(facts) = facts {
12202            let mut left = COMMITS_MAX_CHARS;
12203            for (i, (subject, text)) in facts.commits.iter().enumerate() {
12204                let mut entry = format!("- {}\n", markdown_safe(subject));
12205                for l in markdown_safe(text).lines() {
12206                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
12207                }
12208                if left == 0 {
12209                    body.push_str(&format!(
12210                        "- ... {} more commit(s)\n",
12211                        facts.commits.len() - i
12212                    ));
12213                    break;
12214                }
12215                if entry.len() > left {
12216                    // Even the first commit is cut: one huge body must not
12217                    // push the whole description past GitHub's limit.
12218                    let mut end = left;
12219                    while !entry.is_char_boundary(end) {
12220                        end -= 1;
12221                    }
12222                    entry.truncate(end);
12223                    entry.push_str("\n  ... (truncated)\n");
12224                    left = 0;
12225                } else {
12226                    left -= entry.len();
12227                }
12228                body.push_str(&entry);
12229            }
12230            if !facts.stat.trim().is_empty() {
12231                body.push_str(&format!(
12232                    "\n## Diff stat\n\n```\n{}\n```\n",
12233                    markdown_safe(facts.stat.trim())
12234                ));
12235            }
12236        } else {
12237            match &state.reviewed_commits {
12238                Some(subjects) => {
12239                    for s in subjects {
12240                        body.push_str(&format!("- {}\n", s.trim()));
12241                    }
12242                }
12243                None => {
12244                    // An older run kept only the prompt, with the commit list
12245                    // after its first paragraph.
12246                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
12247                    body.push_str(rest.trim());
12248                    body.push('\n');
12249                }
12250            }
12251        }
12252    } else {
12253        let task = state.instruction.trim();
12254        let task = if task.is_empty() {
12255            "(empty task)"
12256        } else {
12257            task
12258        };
12259        body.push_str(&format!(
12260            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
12261            task.replace("</details>", "&lt;/details&gt;")
12262        ));
12263    }
12264
12265    body.push_str(&format!(
12266        "\n---\nmagi:run/{} magi:candidate-{}\n",
12267        state.id,
12268        winner.to_ascii_lowercase()
12269    ));
12270
12271    PrMessage { title, body }
12272}
12273
12274/// Source imports have already been prepared from structured inputs. Legacy
12275/// inferred references must never instruct a seat to reconstruct those imports.
12276fn seeded_instruction(state: &RunState) -> String {
12277    let mut inputs = Vec::new();
12278    if let Some(start) = &state.sources.start_from {
12279        inputs.push(format!(
12280            "Start from `{}` at `{}` (including ancestry).",
12281            start.token, start.sha
12282        ));
12283    }
12284    for pick in &state.sources.pick_commits {
12285        inputs.push(format!(
12286            "Pick `{}` at `{}` (single commit).",
12287            pick.token, pick.sha
12288        ));
12289    }
12290    if inputs.is_empty() {
12291        state.instruction.clone()
12292    } else {
12293        format!(
12294            "{}\n\n# Explicit source inputs already prepared\n\n{}",
12295            state.instruction,
12296            inputs.join("\n")
12297        )
12298    }
12299}
12300
12301/// Does the winner have no commits ahead of the base it would land on?
12302/// Any failure to find out reads as "not empty": the merge then behaves as it
12303/// always did rather than refusing on a guess.
12304async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
12305    let base = &state.base_branch;
12306    let mut against = base.clone();
12307    if mode == MergeMode::Pr {
12308        // A pull request lands on the remote's base, never the local branch
12309        // of the same name: if that cannot be read, "not empty" is the safe
12310        // answer.
12311        let remote = &state.config.merge.remote;
12312        let tracking = format!("{remote}/{base}");
12313        let fetched = git::fetch(repo, remote, base).await;
12314        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
12315            against = tracking;
12316        } else {
12317            return false;
12318        }
12319    }
12320    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
12321}
12322
12323/// Why nothing was opened for an empty winner, with what the task's own
12324/// references resolved to.
12325fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
12326    let mut detail = format!(
12327        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
12328         nothing to open a pull request for"
12329    );
12330    detail.push_str(
12331        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
12332    );
12333    detail
12334}
12335
12336/// What the `Pr` merge does once it knows whether the branch already has an
12337/// open pull request.
12338#[derive(Debug, PartialEq, Eq)]
12339enum PrPlan {
12340    Create,
12341    Adopt { url: String, title: String },
12342    Stop(String),
12343}
12344
12345/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
12346/// a failed lookup -> stop with the real reason. Never guesses.
12347fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
12348    match found {
12349        Ok(land::OpenPr::None) => PrPlan::Create,
12350        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
12351        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
12352            "several open pull requests exist for this branch, not picking one: {}",
12353            urls.join(" ")
12354        )),
12355        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
12356    }
12357}
12358
12359/// `gh pr create`, returning the PR url.
12360pub(crate) async fn gh_pr_create(
12361    cwd: &Path,
12362    base: &str,
12363    head: &str,
12364    title: &str,
12365    body: &str,
12366) -> Result<String> {
12367    let out = tokio::process::Command::new("gh")
12368        .args([
12369            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
12370        ])
12371        .current_dir(cwd)
12372        .quiet()
12373        .stdin(std::process::Stdio::null())
12374        .output()
12375        .await
12376        .context("spawn gh")?;
12377    if out.status.success() {
12378        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
12379    } else {
12380        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
12381    }
12382}
12383
12384/// Never force deletion of uncommitted work, including unknown legacy trees.
12385async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
12386    match std::fs::symlink_metadata(path) {
12387        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
12388            // A previous cleanup may have removed the tree before a branch
12389            // deletion or state save failed. Unregister it if still present.
12390            git::worktree_remove_clean(repo, path).await?;
12391            return Ok(());
12392        }
12393        Err(e) => return Err(e.into()),
12394        Ok(_) => {}
12395    }
12396    if !git::is_clean(path).await? {
12397        bail!(
12398            "worktree has uncommitted work; refusing removal: {}",
12399            path.display()
12400        );
12401    }
12402    if !git::worktree_remove_clean(repo, path).await? {
12403        bail!("worktree removal refused: {}", path.display());
12404    }
12405    Ok(())
12406}
12407
12408/// Tear a run's worktrees and branches down.
12409///
12410/// `home` is where the updated `run.json` is saved (via
12411/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
12412/// a housekeeping pass already has its own honest `home` handed to it, and
12413/// falling through to the global here would write back through whichever
12414/// directory some other process or test pinned into that `OnceLock` first,
12415/// not the one the caller actually resolved its `runs` and `state` from.
12416pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
12417    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
12418        bail!("refusing fold: {reason}");
12419    }
12420    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
12421        bail!("candidate preservation incomplete; refusing cleanup");
12422    }
12423    // Serialize ordinary folding with the same starts/resumes as target-only
12424    // eviction. Otherwise a second janitor could remove the whole worktree
12425    // around a live output borrower even though target eviction was fenced.
12426    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
12427    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
12428    let _driver = crate::driver::maintenance(home, &state.id)?;
12429    let mut output_guards = Vec::new();
12430    for entry in std::fs::read_dir(state.worktree_root())
12431        .into_iter()
12432        .flatten()
12433        .flatten()
12434    {
12435        if entry.path().join(".git").is_file() {
12436            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
12437        }
12438    }
12439    let repo = state.repo.clone();
12440    // CLI, Web and automatic cleanup all enter here, independently of
12441    // Runner::fold_losers. Protect deletion and shared config release too.
12442    let wait = repository_wait(state, None);
12443    let _coordination = match coordinate_operation(state, &repo, &wait).await {
12444        Ok(lease) => lease,
12445        Err(e) => {
12446            state.save_under(home)?;
12447            return Err(e);
12448        }
12449    };
12450    // Confirmed merges dispose of their owned sources after durable evidence
12451    // capture. Preflight every source before removing anything.
12452    let _source_outputs = if state.status == RunStatus::Merged
12453        && state.merge.as_ref().is_some_and(|merge| merge.ok)
12454    {
12455        verify_merged_fold_sources(state, drop_winner, home).await?
12456    } else {
12457        Vec::new()
12458    };
12459    let root = state.worktree_root();
12460    let winner = state.tally.as_ref().map(|t| t.winner);
12461    let mut removed = Vec::new();
12462
12463    for i in 0..state.candidates.len() {
12464        let c = state.candidates[i].clone();
12465        let is_winner = Some(c.label) == winner;
12466        if is_winner && !drop_winner {
12467            continue;
12468        }
12469        let existed = c.worktree.exists();
12470        // A vanished directory can still have a Git registration that keeps
12471        // its branch checked out. Remove that registration before the branch.
12472        remove_preserved_worktree(&repo, &c.worktree).await?;
12473        if existed {
12474            removed.push(c.worktree.to_string_lossy().into_owned());
12475        }
12476        // A branch handed to a later run (and its pull request) is not this
12477        // run's to delete.
12478        let handed_over = state.released_branches.contains(&c.branch);
12479        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12480            if !git::branch_delete(&repo, &c.branch).await? {
12481                bail!("candidate branch removal refused");
12482            }
12483            removed.push(c.branch.clone());
12484        }
12485        state.candidates[i].folded = true;
12486    }
12487
12488    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12489        let path = name.path();
12490        let keep = !drop_winner
12491            && winner.is_some_and(|w| {
12492                path.file_name()
12493                    .is_some_and(|n| n == format!("cand-{w}").as_str())
12494            });
12495        if keep {
12496            continue;
12497        }
12498        remove_preserved_worktree(&repo, &path).await?;
12499        removed.push(path.to_string_lossy().into_owned());
12500    }
12501
12502    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
12503    // judge worktrees, so once the loop above has cleared all of them out,
12504    // the parent is a bare directory nobody else was ever going to remove -
12505    // git only ever managed what was inside it. Left alone, one of these
12506    // accumulates per fully-folded run; the operator's own machine had 74.
12507    // `remove_if_empty` re-checks rather than assuming: a run whose winner
12508    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
12509    // so does anything a run never claimed that happens to share the bay.
12510    remove_if_empty(&root);
12511
12512    if state.enabled_worktree_config && drop_winner {
12513        // A release, not a raw disable: some sibling run in this repository
12514        // may still hold its own reference (see `git::acquire_worktree_config`),
12515        // and only the last release actually turns the setting back off.
12516        git::release_worktree_config(&repo).await.ok();
12517        state.enabled_worktree_config = false;
12518    }
12519    state.save_under(home)?;
12520    Ok(removed)
12521}
12522
12523async fn verify_merged_fold_sources(
12524    state: &RunState,
12525    drop_winner: bool,
12526    home: &Path,
12527) -> Result<Vec<std::fs::File>> {
12528    let winner = state.tally.as_ref().map(|t| t.winner);
12529    let mut heads = BTreeSet::new();
12530    for candidate in &state.candidates {
12531        if Some(candidate.label) == winner && !drop_winner {
12532            continue;
12533        }
12534        anyhow::ensure!(
12535            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12536            "candidate recovery source is outside the inventoried run root: {}",
12537            candidate.worktree.display()
12538        );
12539        match std::fs::symlink_metadata(&candidate.worktree) {
12540            Ok(_) => {}
12541            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12542            Err(error) => return Err(error.into()),
12543        }
12544        if !state.released_branches.contains(&candidate.branch)
12545            && git::branch_exists(&state.repo, &candidate.branch).await?
12546        {
12547            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12548        }
12549    }
12550    let mut cargo_guards = Vec::new();
12551    let entries = match std::fs::read_dir(state.worktree_root()) {
12552        Ok(entries) => Some(entries),
12553        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12554        Err(error) => return Err(error.into()),
12555    };
12556    for entry in entries.into_iter().flatten() {
12557        let path = entry?.path();
12558        let metadata = std::fs::symlink_metadata(&path)?;
12559        anyhow::ensure!(
12560            metadata.is_dir() && !metadata.file_type().is_symlink(),
12561            "non-directory or symlink recovery source retained: {}",
12562            path.display()
12563        );
12564        if !drop_winner
12565            && winner.is_some_and(|w| {
12566                path.file_name()
12567                    .is_some_and(|name| name == format!("cand-{w}").as_str())
12568            })
12569        {
12570            continue;
12571        }
12572        anyhow::ensure!(
12573            path.join(".git").is_file(),
12574            "unregistered recovery source retained: {}",
12575            path.display()
12576        );
12577        anyhow::ensure!(
12578            git::git(
12579                &path,
12580                &[
12581                    "status",
12582                    "--porcelain",
12583                    "--untracked-files=all",
12584                    "--ignored",
12585                    "--",
12586                    ".",
12587                    ":(exclude)target"
12588                ]
12589            )
12590            .await?
12591            .is_empty(),
12592            "source changes or ignored recovery files retained: {}",
12593            path.display()
12594        );
12595        match std::fs::symlink_metadata(path.join("target")) {
12596            Ok(_) => {
12597                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
12598            }
12599            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12600            Err(error) => return Err(error.into()),
12601        }
12602        for checkpoint in [
12603            "rebase-merge",
12604            "rebase-apply",
12605            "sequencer",
12606            "CHERRY_PICK_HEAD",
12607            "MERGE_HEAD",
12608            "REBASE_HEAD",
12609            "REVERT_HEAD",
12610            "BISECT_LOG",
12611            "AUTO_MERGE",
12612            "MERGE_MSG",
12613        ] {
12614            let location = git::git(
12615                &path,
12616                &[
12617                    "rev-parse",
12618                    "--path-format=absolute",
12619                    "--git-path",
12620                    checkpoint,
12621                ],
12622            )
12623            .await?;
12624            anyhow::ensure!(
12625                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
12626                if e.kind() == std::io::ErrorKind::NotFound),
12627                "Git recovery checkpoint retained: {checkpoint}"
12628            );
12629        }
12630        heads.insert(git::rev_parse(&path, "HEAD").await?);
12631    }
12632    crate::fold_evidence::preserve(state, home, &heads).await?;
12633
12634    Ok(cargo_guards)
12635}
12636
12637/// Remove `dir` if it exists and has nothing in it.
12638///
12639/// Best-effort and silent by design: a directory that is not empty (a run
12640/// whose winner is still parked there, a stray file some other process left)
12641/// is exactly the case this must refuse, and a directory that is already gone
12642/// is not a failure worth reporting either. `std::fs::remove_dir` itself
12643/// already refuses a non-empty directory, so the emptiness check below is
12644/// belt, not suspenders - it is what keeps this from ever attempting the
12645/// removal in the case that matters, rather than trusting `remove_dir`'s
12646/// error path to have no side effects if it ever changed.
12647fn remove_if_empty(dir: &Path) {
12648    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
12649        std::fs::remove_dir(dir).ok();
12650    }
12651}
12652
12653/// Severity of the worst open finding in the last review round, for reporting.
12654pub fn worst_open(state: &RunState) -> Option<Severity> {
12655    state
12656        .reviews
12657        .last()?
12658        .reviews
12659        .iter()
12660        .flat_map(|r| r.findings.iter())
12661        .map(|f| f.severity)
12662        .max()
12663}
12664
12665/// `[worktree] setup` hides its products per worktree, which needs
12666/// `extensions.worktreeConfig`. Taken once per run and released with the run's
12667/// worktrees, like the hook's; every place that runs setup calls this first.
12668async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12669    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12670        git::acquire_worktree_config(repo).await?;
12671        state.enabled_worktree_config = true;
12672    }
12673    Ok(())
12674}
12675
12676#[cfg(test)]
12677mod tests {
12678    #[tokio::test]
12679    async fn launcher_base_fetch_requires_remote_success() {
12680        let tmp = tempfile::tempdir().unwrap();
12681        let repo = tmp.path().join("repo");
12682        let remote = tmp.path().join("remote");
12683        std::fs::create_dir(&repo).unwrap();
12684        std::fs::create_dir(&remote).unwrap();
12685        git::git(&remote, &["init", "--bare", "-b", "main"])
12686            .await
12687            .unwrap();
12688        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12689        git::git(
12690            &repo,
12691            &[
12692                "-c",
12693                "user.name=Fixture",
12694                "-c",
12695                "user.email=fixture@example.invalid",
12696                "commit",
12697                "--allow-empty",
12698                "-m",
12699                "fixture",
12700            ],
12701        )
12702        .await
12703        .unwrap();
12704        git::git(
12705            &repo,
12706            &["remote", "add", "origin", remote.to_str().unwrap()],
12707        )
12708        .await
12709        .unwrap();
12710        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12711        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12712        assert_eq!(
12713            resolve_base(&repo, "main", "origin", None)
12714                .await
12715                .unwrap()
12716                .commit,
12717            head
12718        );
12719        #[cfg(unix)]
12720        {
12721            let policy: crate::config::Coordination = toml::from_str(
12722                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12723            ).unwrap();
12724            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12725            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12726            let held = crate::repo_lock::SeatLease::operation_with(
12727                &repo,
12728                Some(&policy),
12729                &crate::repo_lock::WaitPolicy::default(),
12730            )
12731            .await
12732            .unwrap();
12733            assert!(held.is_some());
12734            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12735            tokio::pin!(fetch);
12736            assert!(
12737                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12738                    .await
12739                    .is_err()
12740            );
12741            assert!(
12742                !repo.join(".git/FETCH_HEAD").exists(),
12743                "no fetch before admission"
12744            );
12745            drop(held);
12746            let fetched = fetch.await.unwrap();
12747            assert_eq!(fetched.commit, head);
12748            assert_eq!(fetched.remote, "origin");
12749            assert_eq!(fetched.branch, "main");
12750        }
12751        // A surviving tracking ref is not successful fetch evidence.
12752        git::git(
12753            &repo,
12754            &[
12755                "remote",
12756                "set-url",
12757                "origin",
12758                tmp.path().join("missing").to_str().unwrap(),
12759            ],
12760        )
12761        .await
12762        .unwrap();
12763        let error = resolve_base(&repo, "main", "origin", None)
12764            .await
12765            .unwrap_err()
12766            .to_string();
12767        assert!(error.contains("cannot read origin/main"));
12768        assert!(error.contains("refusing to branch off"));
12769    }
12770
12771    #[test]
12772    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12773        let leaked = vec!["chore(deps): update a crate".to_owned()];
12774        let own = "fix(daemon): apply a chosen action";
12775        assert!(should_retitle("", own, &leaked));
12776        assert!(should_retitle(
12777            &format!("{REVIEW_PROMPT_OPENING} `x`"),
12778            own,
12779            &leaked
12780        ));
12781        assert!(should_retitle(
12782            "chore: land candidate A of run 1",
12783            own,
12784            &leaked
12785        ));
12786        assert!(should_retitle(
12787            "magi: candidate A (uncommitted work)",
12788            own,
12789            &leaked
12790        ));
12791        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12792        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12793        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12794    }
12795
12796    #[test]
12797    fn pr_merge_plan_creates_adopts_or_stops() {
12798        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12799        assert_eq!(
12800            pr_merge_plan(Ok(land::OpenPr::One {
12801                url: "u".into(),
12802                title: "t".into()
12803            })),
12804            PrPlan::Adopt {
12805                url: "u".into(),
12806                title: "t".into()
12807            }
12808        );
12809        let PrPlan::Stop(many) =
12810            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12811        else {
12812            panic!("many must stop");
12813        };
12814        assert!(many.contains('a') && many.contains('b'));
12815        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12816            panic!("a failed lookup must stop");
12817        };
12818        assert!(err.contains("bad token"));
12819    }
12820
12821    use super::*;
12822    use crate::run::GateStatus;
12823    use std::collections::BTreeMap;
12824    use std::time::Duration;
12825
12826    fn conductor() -> AgentSpec {
12827        AgentSpec {
12828            id: "conductor".to_owned(),
12829            kind: crate::config::AgentKind::Command,
12830            model: None,
12831            command: vec!["true".to_owned()],
12832            extra_args: Vec::new(),
12833            env: BTreeMap::new(),
12834            prompt_delivery: None,
12835            max_turn_secs: None,
12836        }
12837    }
12838
12839    fn spec(id: &str) -> AgentSpec {
12840        AgentSpec {
12841            id: id.to_owned(),
12842            kind: crate::config::AgentKind::Command,
12843            model: None,
12844            command: vec!["true".to_owned()],
12845            extra_args: Vec::new(),
12846            env: BTreeMap::new(),
12847            prompt_delivery: None,
12848            max_turn_secs: None,
12849        }
12850    }
12851
12852    fn ids(xs: &[&str]) -> BTreeSet<String> {
12853        xs.iter().map(|s| (*s).to_owned()).collect()
12854    }
12855
12856    #[test]
12857    fn pick_successor_skips_an_agent_another_seat_holds() {
12858        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
12859        let roster = [spec("a"), spec("b"), spec("c")];
12860        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12861        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12862    }
12863
12864    #[test]
12865    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12866        // The other seat started on c but was handed to d; c is free again.
12867        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12868        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12869        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12870        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12871        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12872    }
12873
12874    #[test]
12875    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12876        let roster = [spec("a"), spec("b")];
12877        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12878        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12879        let carried = ids(&["b"]);
12880        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12881        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12882    }
12883
12884    #[test]
12885    fn pick_successor_returns_none_without_any_untried_successor() {
12886        let roster = [spec("a"), spec("b")];
12887        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12888        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12889        let carried = ids(&["a"]);
12890        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12891    }
12892
12893    #[test]
12894    fn pick_successor_rescue_avoids_another_seats_occupant() {
12895        // b is a carried failure and free; a is held by the other seat.
12896        let roster = [spec("a"), spec("b"), spec("c")];
12897        let carried = ids(&["b", "c"]);
12898        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12899        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12900    }
12901
12902    #[test]
12903    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12904        let roster = [spec("a"), spec("b"), spec("c")];
12905        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12906        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12907    }
12908
12909    #[test]
12910    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12911        let roster = [spec("a"), spec("b"), spec("c")];
12912        let failed = ids(&["a", "b", "c"]);
12913        // Rescue looks at the whole roster, once per id, then runs out.
12914        let mut tried = ids(&["b"]);
12915        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12916        assert_eq!(first.id, "a");
12917        tried.insert(first.id.clone());
12918        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12919        assert_eq!(second.id, "c");
12920        tried.insert(second.id.clone());
12921        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12922    }
12923
12924    #[test]
12925    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12926        let roster = [spec("a"), spec("b")];
12927        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12928        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12929    }
12930
12931    #[test]
12932    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12933        let roster = [spec("a"), spec("b"), spec("c")];
12934        let h = SeatHistory {
12935            failed: ids(&["a"]),
12936            last_ok: Some("b".to_owned()),
12937            last_fail: None,
12938        };
12939        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12940        // A last answerer that left the roster, or later failed, is ignored.
12941        let gone = SeatHistory {
12942            last_ok: Some("zzz".to_owned()),
12943            ..h.clone()
12944        };
12945        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12946        let failed = SeatHistory {
12947            failed: ids(&["a", "b"]),
12948            last_ok: Some("b".to_owned()),
12949            last_fail: None,
12950        };
12951        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12952    }
12953
12954    #[test]
12955    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12956        let first = SeatState::new("review-1", "alpha", 7);
12957        let next = handover_seat("review-1", "gamma", 7);
12958        assert_ne!(first.claude_session, next.claude_session);
12959    }
12960
12961    #[test]
12962    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12963        let mut state = state_with_summary("x", "y");
12964        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12965        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12966        assert_ne!(a.claude_session, b.claude_session);
12967    }
12968
12969    #[test]
12970    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12971        let roster = [spec("a"), spec("b")];
12972        let h = SeatHistory {
12973            failed: ids(&["a", "b"]),
12974            ..SeatHistory::default()
12975        };
12976        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12977        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12978        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12979    }
12980
12981    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12982    // fallback loop depends on to terminate: it must walk forward from the
12983    // seat's own position, never restart at the front of the roster, and it
12984    // must never hand back an id already tried, however many times that id
12985    // happens to appear.
12986
12987    #[test]
12988    fn failure_signature_ignores_numbers_and_paths() {
12989        assert_eq!(
12990            failure_signature("exited with Some(2) and no usable output"),
12991            failure_signature("exited with Some(137) and no usable output")
12992        );
12993        assert_eq!(
12994            failure_signature("process exited with no usable output (exit code 2)"),
12995            failure_signature("process exited with no usable output (exit code 137)")
12996        );
12997        for (old, new) in [
12998            (
12999                "exited with Some(2) and no usable output",
13000                "process exited with no usable output (exit code 2)",
13001            ),
13002            (
13003                "exited with None and no usable output",
13004                "process exited with no usable output (exit code unavailable)",
13005            ),
13006        ] {
13007            let previous = FailClass::Other(failure_signature(old));
13008            let current = FailClass::Other(failure_signature(new));
13009            assert_eq!(previous, current, "resumption retains the failure class");
13010            assert!(!should_hand_over(Some(&previous), &current));
13011        }
13012        assert_eq!(
13013            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13014            failure_signature("cannot open /var/x.txt: denied")
13015        );
13016        assert_ne!(failure_signature("boom"), failure_signature("bang"));
13017    }
13018
13019    #[test]
13020    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13021        let other = FailClass::Other("x".into());
13022        assert!(should_hand_over(None, &FailClass::Quota));
13023        assert!(should_hand_over(Some(&other), &FailClass::Quota));
13024        assert!(should_hand_over(
13025            Some(&FailClass::Timeout),
13026            &FailClass::Timeout
13027        ));
13028        assert!(should_hand_over(None, &other));
13029        assert!(!should_hand_over(Some(&other), &other));
13030        assert!(should_hand_over(
13031            Some(&other),
13032            &FailClass::Other("y".into())
13033        ));
13034        // A quota or timeout in between ends the run of identical failures.
13035        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13036        assert!(should_hand_over(Some(&FailClass::Quota), &other));
13037    }
13038
13039    #[test]
13040    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13041        let a = SeatState::new("judge-1", "alpha", 7);
13042        let b = handover_seat("judge-1", "beta", 7);
13043        assert_ne!(a.claude_session, b.claude_session);
13044        assert_eq!(b.turns, 0);
13045    }
13046
13047    #[test]
13048    fn a_timeout_is_classified_apart_from_other_failures() {
13049        assert_eq!(
13050            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13051            Some(FailClass::Timeout)
13052        );
13053        assert!(matches!(
13054            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13055            Some(FailClass::Other(_))
13056        ));
13057    }
13058
13059    #[test]
13060    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13061        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13062        let tried = BTreeSet::from(["beta".to_owned()]);
13063        // beta sits at index 1; the next candidate is gamma, never alpha —
13064        // which is very likely a different candidate slot's own agent.
13065        let next = next_untried_in_roster(&roster, 1, &tried);
13066        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13067    }
13068
13069    #[test]
13070    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13071        let roster = vec![spec("alpha"), spec("beta")];
13072        let tried = BTreeSet::from(["beta".to_owned()]);
13073        // beta is the roster's last entry: nothing follows it, and alpha —
13074        // earlier in the roster, almost certainly a different candidate
13075        // slot's own agent — must not be reached by wrapping back to it.
13076        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13077    }
13078
13079    #[test]
13080    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13081        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13082        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13083        // beta (index 1) and gamma (index 2, the only entry after it) have
13084        // both been tried; alpha (index 0) never has, but it comes before
13085        // beta's own position, so there is nothing further for this seat.
13086        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13087    }
13088
13089    #[test]
13090    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13091        let roster = vec![spec("a"), spec("a"), spec("b")];
13092        let tried = BTreeSet::from(["a".to_owned()]);
13093        let next = next_untried_in_roster(&roster, 0, &tried);
13094        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13095    }
13096
13097    #[test]
13098    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13099        let roster = vec![spec("a"), spec("b")];
13100        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13101        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13102    }
13103
13104    #[test]
13105    fn remove_if_empty_only_ever_takes_a_bare_directory() {
13106        let dir = tempfile::tempdir().unwrap();
13107        let bay = dir.path().join("ffff");
13108
13109        // Not there yet: nothing to do, nothing to panic on.
13110        remove_if_empty(&bay);
13111        assert!(!bay.exists());
13112
13113        // Something still inside - the winner's worktree, or a stray file -
13114        // keeps the directory standing.
13115        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13116        remove_if_empty(&bay);
13117        assert!(bay.exists(), "non-empty directory must survive");
13118
13119        // Once the last entry is gone, so is the directory itself.
13120        std::fs::remove_dir(bay.join("cand-A")).unwrap();
13121        remove_if_empty(&bay);
13122        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13123    }
13124
13125    // `round_is_clean` is the exact decision this task fixed: a round with a
13126    // seat that never answered must not read the same as a round every seat
13127    // actually reviewed. These are deterministic and process-free by design —
13128    // the equivalent end-to-end check (a real reviewer timing out under a
13129    // live graph run) is a genuine race against wall-clock contention, and a
13130    // spawn slow enough to blow even a generous budget under a loaded test
13131    // run must not turn this specific regression check flaky.
13132
13133    #[test]
13134    fn a_full_panel_that_found_nothing_is_clean() {
13135        assert!(round_is_clean(
13136            0,
13137            true,
13138            2,
13139            2,
13140            0,
13141            IncompleteReviewPolicy::Block
13142        ));
13143    }
13144
13145    #[test]
13146    fn a_missing_seat_is_never_clean_under_the_default_policy() {
13147        assert!(!round_is_clean(
13148            0,
13149            true,
13150            1,
13151            2,
13152            0,
13153            IncompleteReviewPolicy::Block
13154        ));
13155    }
13156
13157    #[test]
13158    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13159        assert!(!round_is_clean(
13160            1,
13161            true,
13162            1,
13163            2,
13164            0,
13165            IncompleteReviewPolicy::Warn
13166        ));
13167    }
13168
13169    #[test]
13170    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13171        assert!(round_is_clean(
13172            0,
13173            true,
13174            1,
13175            2,
13176            0,
13177            IncompleteReviewPolicy::Warn
13178        ));
13179    }
13180
13181    #[test]
13182    fn a_full_panel_with_an_open_finding_is_not_clean() {
13183        assert!(!round_is_clean(
13184            1,
13185            true,
13186            2,
13187            2,
13188            0,
13189            IncompleteReviewPolicy::Block
13190        ));
13191    }
13192
13193    #[test]
13194    fn a_full_panel_with_a_red_e2e_is_not_clean() {
13195        assert!(!round_is_clean(
13196            0,
13197            false,
13198            2,
13199            2,
13200            0,
13201            IncompleteReviewPolicy::Block
13202        ));
13203    }
13204
13205    // The stall this task closes: under the default `block` policy, a seat
13206    // missing only because it was rate limited must not force a wait for a
13207    // session limit that will not lift by the next round. `round_is_clean`
13208    // is where that quorum carve-out lives; the review loop around it never
13209    // changes what a reviewer's vote or a finding's severity means.
13210
13211    #[test]
13212    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13213        // 1 of 2 answered, and the one missing was quota'd — the exact
13214        // "review-2 rate limited (quota)" shape from the field report.
13215        assert!(round_is_clean(
13216            0,
13217            true,
13218            1,
13219            2,
13220            1,
13221            IncompleteReviewPolicy::Block
13222        ));
13223    }
13224
13225    #[test]
13226    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13227        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
13228        // not a quota loss (`quota_missing` stays 0) — worth another try.
13229        assert!(!round_is_clean(
13230            0,
13231            true,
13232            1,
13233            2,
13234            0,
13235            IncompleteReviewPolicy::Block
13236        ));
13237    }
13238
13239    #[test]
13240    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
13241        assert!(!round_is_clean(
13242            1,
13243            true,
13244            1,
13245            2,
13246            1,
13247            IncompleteReviewPolicy::Block
13248        ));
13249        assert!(!round_is_clean(
13250            0,
13251            false,
13252            1,
13253            2,
13254            1,
13255            IncompleteReviewPolicy::Block
13256        ));
13257    }
13258
13259    #[test]
13260    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
13261        // Every seat quota'd, nobody answered: there is no panel to decide
13262        // on, so this must fall through to the existing block-and-retry
13263        // fallback rather than call an unreviewed patch clean.
13264        assert!(!round_is_clean(
13265            0,
13266            true,
13267            0,
13268            2,
13269            2,
13270            IncompleteReviewPolicy::Block
13271        ));
13272    }
13273
13274    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
13275        CommandOutcome {
13276            command: "test".to_owned(),
13277            code,
13278            output_tail: String::new(),
13279            duration_ms: 0,
13280            resource_blocked,
13281        }
13282    }
13283
13284    #[test]
13285    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
13286        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
13287        assert!(
13288            !verify_inconclusive(&[outcome(Some(1), false)]),
13289            "an ordinary failure is still evidence about the patch"
13290        );
13291        assert!(verify_inconclusive(&[outcome(None, true)]));
13292        assert!(
13293            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
13294            "one inconclusive outcome taints the whole batch"
13295        );
13296        assert!(!verify_inconclusive(&[]));
13297    }
13298
13299    #[tokio::test]
13300    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
13301        // Alive for the first two checks, then dead - confirms the loop
13302        // actually re-polls rather than deciding once and sleeping out the
13303        // ceiling regardless.
13304        let calls = std::sync::atomic::AtomicUsize::new(0);
13305        let started = Instant::now();
13306        wait_for_pids_with(
13307            &[123],
13308            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
13309            Duration::from_millis(5),
13310            Duration::from_secs(5),
13311        )
13312        .await;
13313        assert!(
13314            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
13315            "must keep checking rather than deciding on the first answer"
13316        );
13317        assert!(
13318            started.elapsed() < Duration::from_secs(1),
13319            "must return the moment it is confirmed dead, not wait out the ceiling"
13320        );
13321    }
13322
13323    #[tokio::test]
13324    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
13325        let started = Instant::now();
13326        wait_for_pids_with(
13327            &[123],
13328            |_| true, // never reports dead
13329            Duration::from_millis(5),
13330            Duration::from_millis(30),
13331        )
13332        .await;
13333        let elapsed = started.elapsed();
13334        assert!(
13335            elapsed >= Duration::from_millis(30),
13336            "must not give up before its own ceiling: {elapsed:?}"
13337        );
13338        assert!(
13339            elapsed < Duration::from_secs(1),
13340            "must not wait past its own ceiling either: {elapsed:?}"
13341        );
13342    }
13343
13344    #[tokio::test]
13345    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
13346        let started = Instant::now();
13347        wait_for_pids_with(
13348            &[],
13349            |_| true,
13350            Duration::from_secs(5),
13351            Duration::from_secs(5),
13352        )
13353        .await;
13354        assert!(
13355            started.elapsed() < Duration::from_millis(200),
13356            "an empty pid list has nothing to confirm"
13357        );
13358    }
13359
13360    // `review_conclusion` is the exact decision the review hand-off task
13361    // fixed: a round budget spent (or a tree that stopped moving) must not
13362    // collapse into `Blocked` regardless of what verification actually
13363    // said. Deterministic and process-free for the same reason the
13364    // `round_is_clean` family above is.
13365    fn review_round(
13366        clean: bool,
13367        blocking: usize,
13368        answered: usize,
13369        expected: usize,
13370        progressed: bool,
13371        e2e_ok: bool,
13372    ) -> ReviewRound {
13373        ReviewRound {
13374            round: 1,
13375            head: "h".to_owned(),
13376            verified_head: None,
13377            verified_at: None,
13378            reviews: Vec::new(),
13379            e2e: vec![CommandOutcome {
13380                command: "test".to_owned(),
13381                code: Some(if e2e_ok { 0 } else { 1 }),
13382                output_tail: String::new(),
13383                duration_ms: 0,
13384                resource_blocked: false,
13385            }],
13386            verify_retried: false,
13387            e2e_deferred: false,
13388            e2e_defer_reason: None,
13389            fix: None,
13390            blocking,
13391            answered,
13392            expected,
13393            clean,
13394            progressed,
13395            vote_split: false,
13396            reconsideration: Vec::new(),
13397            verdict: None,
13398        }
13399    }
13400
13401    #[test]
13402    fn review_conclusion_is_none_when_nothing_has_run() {
13403        assert_eq!(review_conclusion(&[], 3), None);
13404    }
13405
13406    #[test]
13407    fn review_conclusion_is_none_while_rounds_remain() {
13408        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
13409        assert_eq!(review_conclusion(&rounds, 3), None);
13410    }
13411
13412    #[test]
13413    fn review_conclusion_is_gating_once_a_round_is_clean() {
13414        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
13415        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
13416    }
13417
13418    #[test]
13419    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
13420        let rounds = vec![
13421            review_round(false, 1, 2, 2, true, true),
13422            review_round(false, 1, 2, 2, true, true),
13423        ];
13424        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
13425    }
13426
13427    #[test]
13428    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
13429        let rounds = vec![
13430            review_round(false, 1, 2, 2, true, true),
13431            review_round(false, 1, 2, 2, true, false),
13432        ];
13433        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
13434    }
13435
13436    #[test]
13437    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
13438        // Magi never got a command to run against this round's own head — a
13439        // resource-blocked attempt, not a red one — so this must never
13440        // settle on `Blocked` the way a genuine e2e failure would. `None`
13441        // here is what tells `Runner::review_loop` to retry the check
13442        // itself rather than trust this cheap recomputation with a verdict
13443        // it cannot actually produce.
13444        let mut blocked = review_round(false, 1, 2, 2, true, false);
13445        blocked.e2e[0].resource_blocked = true;
13446        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
13447        assert_eq!(review_conclusion(&rounds, 2), None);
13448    }
13449
13450    #[test]
13451    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
13452        // Missing input, not a verified tree — never a hand-off candidate.
13453        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
13454        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
13455    }
13456
13457    #[test]
13458    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
13459        let rounds = vec![
13460            review_round(false, 1, 2, 2, false, true),
13461            review_round(false, 1, 2, 2, false, true),
13462        ];
13463        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
13464    }
13465
13466    fn secs(n: u64) -> Duration {
13467        Duration::from_secs(n)
13468    }
13469
13470    /// A throwaway repo with one commit on `main`, for tests that need `merge`
13471    /// to make real (and, if it runs at all, real*ly fail*) git calls.
13472    #[tokio::test]
13473    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13474        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
13475        if std::env::var_os(CHILD).is_none() {
13476            // Other tests reconcile every run in the process-global home,
13477            // briefly taking owner locks. This ancestry test expects an idle
13478            // predecessor, so give it its own process and pinned test home.
13479            let output = std::process::Command::new(std::env::current_exe().unwrap())
13480                .args([
13481                    "--exact",
13482                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
13483                    "--nocapture",
13484                ])
13485                .env(CHILD, "1")
13486                .env_remove("MAGI_WORKER_HOME")
13487                .env_remove("MAGI_WORKER_ID")
13488                .output()
13489                .unwrap();
13490            assert!(
13491                output.status.success()
13492                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
13493                "{}\n{}",
13494                String::from_utf8_lossy(&output.stdout),
13495                String::from_utf8_lossy(&output.stderr)
13496            );
13497            return;
13498        }
13499        ask_test_home();
13500        let tmp = tempfile::tempdir().unwrap();
13501        let repo = tmp.path().join("repo");
13502        std::fs::create_dir_all(&repo).unwrap();
13503        init_repo(&repo);
13504        let g = |args: &[&str]| {
13505            let out = std::process::Command::new("git")
13506                .args(args)
13507                .current_dir(&repo)
13508                .quiet()
13509                .output()
13510                .unwrap();
13511            assert!(
13512                out.status.success(),
13513                "{}",
13514                String::from_utf8_lossy(&out.stderr)
13515            );
13516        };
13517        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13518        g(&["checkout", "-b", "historical"]);
13519        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13520        g(&["add", "."]);
13521        g(&["commit", "-m", "old alternative"]);
13522        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13523        g(&["checkout", "-b", "magi/task-selected", &base]);
13524        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13525        g(&["add", "."]);
13526        g(&["commit", "-m", "measurement infrastructure"]);
13527        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13528        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13529        g(&["add", "."]);
13530        g(&["commit", "-m", "dependent implementation"]);
13531        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13532        g(&["checkout", "main"]);
13533        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13534        g(&["add", "."]);
13535        g(&["commit", "-m", "unrelated base change"]);
13536        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13537        let origin = tmp.path().join("origin.git");
13538        g(&[
13539            "clone",
13540            "--bare",
13541            repo.to_str().unwrap(),
13542            origin.to_str().unwrap(),
13543        ]);
13544        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13545        g(&["fetch", "origin"]);
13546        let selection = crate::pr_recovery::Selection {
13547            url: "https://example.test/project/pull/1".into(),
13548            branch: "magi/task-selected".into(),
13549            head: head.clone(),
13550            source_head: None,
13551            source_run: None,
13552            source_proof: None,
13553            disposition: Some(crate::pr_recovery::Disposition::Review),
13554            instruction: "preserve measured behavior".into(),
13555            approved_at: jiff::Timestamp::now(),
13556        };
13557        let mut template = runner_at(RunStatus::Reviewing);
13558        template.state.repo = repo.clone();
13559        template.state.base_commit = current.clone();
13560        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13561        template.state.origin = Some(crate::run::Origin::queue("selected"));
13562        template.state.pr_head = Some(selection.branch.clone());
13563        template.state.pr_recovery = Some(selection.clone());
13564        let mut runner =
13565            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13566                .await
13567                .unwrap();
13568        let wt = runner.state.winner().unwrap().worktree.clone();
13569        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13570        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13571        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13572        assert!(runner.state.seeds.is_empty());
13573        assert_eq!(runner.state.instruction, "preserve measured behavior");
13574        runner.sync_to_base().await.unwrap();
13575        assert_eq!(
13576            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13577            "prerequisite"
13578        );
13579        assert_eq!(
13580            std::fs::read_to_string(wt.join("implementation")).unwrap(),
13581            "measured implementation"
13582        );
13583        assert_eq!(
13584            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13585            "current main change"
13586        );
13587        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13588        assert_eq!(
13589            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13590            head
13591        );
13592
13593        // A failed selected-source setup stays inspectable, never resumable as a new competition.
13594        let mut partial = runner_at(RunStatus::Reviewing);
13595        partial.state.repo = repo.clone();
13596        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
13597        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
13598            run: Some("exit 1".into()),
13599            ..Default::default()
13600        }];
13601        let selection = runner.state.pr_recovery.clone().unwrap();
13602        partial.state.pr_recovery = Some(selection.clone());
13603        partial.state.pr_head = Some(selection.branch.clone());
13604        partial.state.origin = Some(crate::run::Origin::queue("selected"));
13605        let id = partial.state.id.clone();
13606        let error = match Runner::open_selected(
13607            &repo,
13608            partial.state,
13609            partial.roles,
13610            1,
13611            runner.state.base_commit.clone(),
13612            selection,
13613        )
13614        .await
13615        {
13616            Ok(_) => panic!("setup must fail"),
13617            Err(e) => e,
13618        };
13619        assert_eq!(
13620            error
13621                .downcast_ref::<crate::pr_recovery::StartupFailed>()
13622                .unwrap()
13623                .run,
13624            id
13625        );
13626        let failed = RunState::load(&id).unwrap();
13627        assert_eq!(failed.status, RunStatus::Blocked);
13628        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
13629        assert!(Runner::resume(&id).is_err());
13630        runner.state.status = RunStatus::Ready;
13631        runner.state.driver_exited = true;
13632        runner.state.save().unwrap();
13633        let mut retry = runner_at(RunStatus::Reviewing);
13634        retry.state.repo = repo.clone();
13635        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
13636        let mut reselected = runner.state.pr_recovery.clone().unwrap();
13637        reselected.approved_at = jiff::Timestamp::now();
13638        retry.state.origin = Some(crate::run::Origin::queue("selected"));
13639        retry.state.pr_head = Some(reselected.branch.clone());
13640        retry.state.pr_recovery = Some(reselected.clone());
13641        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
13642            .await
13643            .unwrap();
13644        let restarted = Runner::open_selected(
13645            &repo,
13646            retry.state,
13647            retry.roles,
13648            1,
13649            runner.state.base_commit.clone(),
13650            reselected,
13651        )
13652        .await
13653        .unwrap();
13654        assert_eq!(
13655            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
13656                .await
13657                .unwrap(),
13658            head
13659        );
13660        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13661        assert_eq!(
13662            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13663            head
13664        );
13665    }
13666
13667    #[tokio::test]
13668    #[cfg(unix)]
13669    async fn fallback_and_interrupted_restoration_require_coordination() {
13670        use std::os::fd::AsRawFd as _;
13671        ask_test_home();
13672        let tmp = tempfile::tempdir().unwrap();
13673        let mut runner = registered_runner(tmp.path()).await;
13674        runner.state.config.graph.timeout_resource_wait = 1;
13675        let winner = runner.state.candidates[0].clone();
13676        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13677        let leftover = winner.worktree.join("leftover");
13678        std::fs::write(&leftover, "untrusted edits").unwrap();
13679        let next = tmp.path().join("next-fixer");
13680        let mut failed = spec("first");
13681        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
13682        let mut successor = spec("second");
13683        successor.command = vec![
13684            "sh".into(),
13685            "-c".into(),
13686            format!("touch '{}'; echo done", next.display()),
13687        ];
13688        let attempts = vec![
13689            (failed.clone(), "first".into()),
13690            (successor.clone(), "second".into()),
13691        ];
13692        let lock_path = runner.state.repo.join(".git/updater.lock");
13693        let external = std::fs::File::create(&lock_path).unwrap();
13694        assert_eq!(
13695            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13696            0
13697        );
13698        let artifacts = tmp.path().join("artifacts");
13699        let build = |spec, seat| SeatJob {
13700            spec,
13701            seat,
13702            cwd: winner.worktree.clone(),
13703            prompt: "fixture".into(),
13704            timeout: Duration::from_secs(10),
13705            allow_write: true,
13706            sessions: false,
13707            artifacts: artifacts.clone(),
13708            stem: "fix".into(),
13709            handover: None,
13710        };
13711        let (_, _, out) = runner
13712            .ask_fixer_among(
13713                attempts.clone(),
13714                "graft",
13715                None,
13716                Some((&winner.worktree, &head)),
13717                build,
13718            )
13719            .await;
13720        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13721        assert!(leftover.exists(), "contention must prevent clean");
13722        assert!(!next.exists());
13723        assert!(runner.state.handovers.is_empty());
13724        // Early interrupted-repair rollback uses the same protected route.
13725        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13726            from_run: "earlier".into(),
13727            generation: 1,
13728            mode: crate::recovery::Mode::VerifyRepair,
13729            reasons: Vec::new(),
13730            reviewed_head: None,
13731            rounds: Vec::new(),
13732            open_findings: Vec::new(),
13733            evidence: None,
13734            criteria_inherited: true,
13735            repair: crate::recovery::Repair::Started {
13736                before: head.clone(),
13737            },
13738        });
13739        assert!(
13740            runner
13741                .recovery_repair(&winner)
13742                .await
13743                .unwrap_err()
13744                .is::<crate::repo_lock::ResourceBlocked>()
13745        );
13746        assert!(matches!(
13747            runner.state.recovery.as_ref().unwrap().repair,
13748            crate::recovery::Repair::Started { .. }
13749        ));
13750        assert!(leftover.exists());
13751        drop(external);
13752        runner.recovery_repair(&winner).await.unwrap();
13753        assert!(!leftover.exists());
13754        assert!(matches!(
13755            runner.state.recovery.as_ref().unwrap().repair,
13756            crate::recovery::Repair::Done { .. }
13757        ));
13758        // A hard lock-open failure also leaves HEAD and untracked work intact.
13759        std::fs::remove_file(&lock_path).unwrap();
13760        std::fs::create_dir(&lock_path).unwrap();
13761        std::fs::write(&leftover, "untrusted edits").unwrap();
13762        let (_, _, out) = runner
13763            .ask_fixer_among(
13764                attempts,
13765                "graft",
13766                None,
13767                Some((&winner.worktree, &head)),
13768                build,
13769            )
13770            .await;
13771        assert!(matches!(out, AgentOutcome::Failed(_)));
13772        assert!(leftover.exists());
13773        assert!(!next.exists());
13774        assert_eq!(
13775            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13776            head
13777        );
13778    }
13779
13780    #[cfg(unix)]
13781    async fn registered_runner(root: &Path) -> Runner {
13782        repository_runner(root, crate::config::test_coordination()).await
13783    }
13784
13785    async fn repository_runner(
13786        root: &Path,
13787        coordination: Option<crate::config::Coordination>,
13788    ) -> Runner {
13789        ask_test_home();
13790        let repo = root.join("repo");
13791        std::fs::create_dir_all(&repo).unwrap();
13792        init_repo(&repo);
13793        if coordination.is_some() {
13794            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13795        }
13796        let mut runner = runner_at(RunStatus::Prep);
13797        runner.state.repo = repo.clone();
13798        runner.state.config.coordination = coordination;
13799        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13800        runner.state.config.graph.worktree_root = Some(root.join("wt"));
13801        runner.state.config.graph.advise = false;
13802        runner.roles.implementers = vec![spec("implement")];
13803        runner.roles.judges = vec![spec("judge")];
13804        runner.prep().await.unwrap();
13805        runner
13806    }
13807
13808    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
13809
13810    #[tokio::test]
13811    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
13812        let tmp = tempfile::tempdir().unwrap();
13813        let mut runner = repository_runner(tmp.path(), None).await;
13814        let prompts = runner.state.config.prompts.clone();
13815        let run = runner.state.id.clone();
13816        let ctx = WaveCtx {
13817            carry_seats: false,
13818            pauses: None,
13819            run: &run,
13820            node: "advise",
13821            prompts: &prompts,
13822            cache: None,
13823            round: None,
13824        };
13825        let jobs: Vec<_> = ["alpha", "beta"]
13826            .into_iter()
13827            .map(|id| SeatJob {
13828                spec: scripted(id, AUTH_OUTAGE),
13829                seat: SeatState::new(id, id, 7),
13830                prompt: "propose".into(),
13831                cwd: runner.state.candidates[0].worktree.clone(),
13832                timeout: Duration::from_secs(10),
13833                allow_write: false,
13834                sessions: false,
13835                artifacts: agent::artifacts_dir(&runner.state.dir()),
13836                stem: id.into(),
13837                handover: None,
13838            })
13839            .collect();
13840        let mut losses = Vec::new();
13841        let results = ask_wave_with(
13842            jobs.clone(),
13843            runner.sem.clone(),
13844            2,
13845            &[],
13846            &ctx,
13847            &mut losses,
13848            &mut runner.state,
13849            &|_: &str| Ok(()),
13850        )
13851        .await;
13852        assert!(
13853            results
13854                .iter()
13855                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
13856        );
13857        assert_eq!(runner.state.quota.len(), 2);
13858        assert!(
13859            runner
13860                .state
13861                .quota
13862                .iter()
13863                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
13864        );
13865        assert!(!runner.state.advise_attempted);
13866        runner.stop_for_resource().unwrap();
13867        assert_eq!(runner.state.status, RunStatus::Blocked);
13868        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
13869        assert_eq!(
13870            runner.state.resource_block.as_ref().unwrap().kind,
13871            "agent authentication"
13872        );
13873        // A machine loss followed by an ordinary failure is not evidence
13874        // that the whole node was unavailable, and must not earn a refund.
13875        runner.state.resource_block = None;
13876        runner.state.quota.clear();
13877        let mut mixed = jobs;
13878        mixed[1].spec.command = vec!["false".into()];
13879        let results = ask_wave_with(
13880            mixed,
13881            runner.sem.clone(),
13882            0,
13883            &[],
13884            &ctx,
13885            &mut losses,
13886            &mut runner.state,
13887            &|_: &str| Ok(()),
13888        )
13889        .await;
13890        assert!(results.iter().all(|(_, res, _)| res.is_err()));
13891        assert!(runner.state.resource_block.is_none());
13892        assert_eq!(losses.len(), 1);
13893    }
13894
13895    #[tokio::test]
13896    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
13897        let tmp = tempfile::tempdir().unwrap();
13898        let mut runner = repository_runner(tmp.path(), None).await;
13899        let implementer = scripted("implement", AUTH_OUTAGE);
13900        runner.roles.implementers = vec![implementer.clone()];
13901        runner.state.config.agents = vec![implementer];
13902        runner.implement().await.unwrap();
13903        assert_eq!(runner.state.status, RunStatus::Blocked);
13904        assert_eq!(runner.state.quota.len(), 1);
13905        assert_eq!(
13906            runner.state.quota[0].cause,
13907            agent::QuotaCause::Authentication
13908        );
13909        let candidate = &runner.state.candidates[0];
13910        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
13911        assert_eq!(runner.state.seats["impl-A"].turns, 1);
13912    }
13913
13914    #[tokio::test]
13915    #[cfg(unix)]
13916    async fn parked_implementer_persists_completed_work() {
13917        for registered in [false, true] {
13918            let tmp = tempfile::tempdir().unwrap();
13919            let mut runner = registered_runner(tmp.path()).await;
13920            if !registered {
13921                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13922            }
13923            let marker = tmp.path().join("started");
13924            let release = tmp.path().join("release");
13925            let mut implementer = spec("implement");
13926            implementer.command = vec![
13927                "sh".into(),
13928                "-c".into(),
13929                format!(
13930                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13931                    marker.display(),
13932                    release.display()
13933                ),
13934            ];
13935            runner.roles.implementers = vec![implementer.clone()];
13936            runner.state.config.agents = vec![implementer];
13937            let pause = runner.pause.clone();
13938            let (result, ()) = tokio::join!(runner.implement(), async {
13939                tokio::time::timeout(Duration::from_secs(10), async {
13940                    while !marker.exists() {
13941                        tokio::time::sleep(Duration::from_millis(10)).await;
13942                    }
13943                })
13944                .await
13945                .unwrap();
13946                pause.park();
13947                std::fs::write(&release, "go").unwrap();
13948            });
13949            result.unwrap();
13950            let candidate = &runner.state.candidates[0];
13951            assert!(candidate.commits > 0, "{candidate:?}");
13952            assert!(candidate.summary.contains("saved-result"));
13953            assert_eq!(candidate.files, 1);
13954            let before = candidate.commits;
13955            runner.implement().await.unwrap();
13956            assert_eq!(runner.state.candidates[0].commits, before);
13957        }
13958    }
13959
13960    #[tokio::test]
13961    #[cfg(unix)]
13962    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13963        let tmp = tempfile::tempdir().unwrap();
13964        let mut runner = registered_runner(tmp.path()).await;
13965        let winner = runner.state.candidates[0].clone();
13966        let cache = tmp.path().join("cache");
13967        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13968        runner.state.config.graph.timeout_resource_wait = 0;
13969        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13970        let crate::cache::AcquireOutcome::Acquired(held) =
13971            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13972        else {
13973            panic!("cache must be free")
13974        };
13975        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13976            from_run: "earlier".into(),
13977            generation: 1,
13978            mode: crate::recovery::Mode::VerifyRepair,
13979            reasons: Vec::new(),
13980            reviewed_head: None,
13981            rounds: Vec::new(),
13982            open_findings: Vec::new(),
13983            evidence: None,
13984            criteria_inherited: true,
13985            repair: crate::recovery::Repair::NotStarted,
13986        });
13987        runner.recovery_repair(&winner).await.unwrap();
13988        assert!(matches!(
13989            runner.state.recovery.as_ref().unwrap().repair,
13990            crate::recovery::Repair::WaitingForRestore { .. }
13991        ));
13992        drop(held);
13993        runner.state.resource_block = None;
13994        runner.state.config.graph.timeout_resource_wait = 1;
13995        runner.recovery_repair(&winner).await.unwrap();
13996        assert!(matches!(
13997            runner.state.recovery.as_ref().unwrap().repair,
13998            crate::recovery::Repair::Done { .. }
13999        ));
14000        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14001    }
14002
14003    #[tokio::test]
14004    #[cfg(unix)]
14005    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14006        use std::os::fd::AsRawFd as _;
14007        for graft in [false, true] {
14008            let tmp = tempfile::tempdir().unwrap();
14009            let mut runner = registered_runner(tmp.path()).await;
14010            runner.state.config.graph.timeout_resource_wait = 1;
14011            let winner = runner.state.candidates[0].clone();
14012            let marker = tmp.path().join("fixer-started");
14013            let mut fixer = spec("fixer");
14014            fixer.command = vec![
14015                "sh".into(),
14016                "-c".into(),
14017                format!("echo started >> '{}'; echo done", marker.display()),
14018            ];
14019            runner.roles.fixer = Some(vec![fixer]);
14020            if graft {
14021                let mut loser = winner.clone();
14022                loser.label = 'B';
14023                loser.index = 1;
14024                runner.state.candidates.push(loser);
14025                runner.state.config.graph.graft_runner_up = true;
14026                runner.state.tally = Some(Tally {
14027                    first_choice: BTreeMap::new(),
14028                    borda: BTreeMap::new(),
14029                    winner: winner.label,
14030                    rankings: 1,
14031                    unanimous_initial: true,
14032                    deliberated: false,
14033                    changed_votes: 0,
14034                    unanimous_final: true,
14035                    tie_break: None,
14036                    judges: 1,
14037                    present: 1,
14038                    quorum: 1,
14039                    met_quorum: true,
14040                    uncontested: None,
14041                });
14042                runner.state.judgements.push(crate::run::Judgement {
14043                    judge: 1,
14044                    seat: "judge-1".into(),
14045                    agent: "judge".into(),
14046                    ranking: vec![winner.label, 'B'],
14047                    reasons: BTreeMap::new(),
14048                    confidence: None,
14049                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14050                    order: vec![0, 1],
14051                    failed: None,
14052                    duration_ms: 0,
14053                });
14054            } else {
14055                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14056                    from_run: "earlier".into(),
14057                    generation: 1,
14058                    mode: crate::recovery::Mode::VerifyRepair,
14059                    reasons: Vec::new(),
14060                    reviewed_head: None,
14061                    rounds: Vec::new(),
14062                    open_findings: Vec::new(),
14063                    evidence: None,
14064                    criteria_inherited: true,
14065                    repair: crate::recovery::Repair::NotStarted,
14066                });
14067            }
14068            let reader = crate::repo_lock::SeatLease::for_checkout(
14069                &winner.worktree,
14070                false,
14071                crate::config::test_coordination().as_ref(),
14072            )
14073            .await
14074            .unwrap();
14075            let pause = runner.pause.clone();
14076            let (result, ()) = tokio::join!(
14077                async {
14078                    if graft {
14079                        runner.graft().await
14080                    } else {
14081                        runner.recovery_repair(&winner).await
14082                    }
14083                },
14084                async {
14085                    tokio::time::sleep(Duration::from_millis(100)).await;
14086                    pause.park();
14087                }
14088            );
14089            result.unwrap();
14090            assert!(!marker.exists(), "a blocked seat must not spawn");
14091            // Exercise the persisted state, as a binary replacement/resume does.
14092            runner.state = RunState::load(&runner.state.id).unwrap();
14093            if graft {
14094                assert_eq!(
14095                    runner.state.graft.as_ref().unwrap().outcome,
14096                    GraftOutcome::WaitingForRestore
14097                );
14098            } else {
14099                assert!(matches!(
14100                    runner.state.recovery.as_ref().unwrap().repair,
14101                    crate::recovery::Repair::WaitingForRestore { .. }
14102                ));
14103            }
14104            drop(reader);
14105            runner.pause = Pause::new();
14106            runner.state.resource_block = None;
14107            // Deferred restoration must still respect an external lock owner.
14108            let external =
14109                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14110            assert_eq!(
14111                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14112                0
14113            );
14114            let leftover = winner.worktree.join("leftover");
14115            std::fs::write(&leftover, "earlier fallback edits").unwrap();
14116            let result = if graft {
14117                runner.graft().await
14118            } else {
14119                runner.recovery_repair(&winner).await
14120            };
14121            assert!(
14122                result
14123                    .unwrap_err()
14124                    .is::<crate::repo_lock::ResourceBlocked>()
14125            );
14126            assert!(leftover.exists());
14127            assert!(!marker.exists());
14128            drop(external);
14129            runner.state.resource_block = None;
14130            let result = if graft {
14131                runner.graft().await
14132            } else {
14133                runner.recovery_repair(&winner).await
14134            };
14135            result.unwrap();
14136            assert!(!leftover.exists(), "restore must precede retry");
14137            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14138            if graft {
14139                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14140            } else {
14141                assert!(matches!(
14142                    runner.state.recovery.as_ref().unwrap().repair,
14143                    crate::recovery::Repair::Done { .. }
14144                ));
14145            }
14146        }
14147    }
14148
14149    #[tokio::test]
14150    #[cfg(unix)]
14151    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
14152        use std::os::fd::AsRawFd as _;
14153        let tmp = tempfile::tempdir().unwrap();
14154        let mut runner = registered_runner(tmp.path()).await;
14155        runner.state.config.graph.timeout_resource_wait = 1;
14156        let winner = runner.state.candidates[0].clone();
14157        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14158        runner.state.tally = Some(Tally {
14159            first_choice: BTreeMap::new(),
14160            borda: BTreeMap::new(),
14161            winner: winner.label,
14162            rankings: 0,
14163            unanimous_initial: false,
14164            deliberated: false,
14165            changed_votes: 0,
14166            unanimous_final: false,
14167            tie_break: None,
14168            judges: 0,
14169            present: 0,
14170            quorum: 0,
14171            met_quorum: true,
14172            uncontested: Some("fixture".into()),
14173        });
14174        runner.state.graft = Some(GraftRecord {
14175            pre: pre.clone(),
14176            post: None,
14177            lines: None,
14178            limit: None,
14179            agent: None,
14180            outcome: GraftOutcome::Running,
14181            requested: Vec::new(),
14182            adoption: None,
14183            adoption_error: None,
14184        });
14185        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
14186        git::commit_all(&winner.worktree, "unfinished graft")
14187            .await
14188            .unwrap();
14189        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14190        let leftover = winner.worktree.join("leftover");
14191        std::fs::write(&leftover, "untrusted edits").unwrap();
14192        let path = runner.state.repo.join(".git/updater.lock");
14193        let external = std::fs::File::create(&path).unwrap();
14194        assert_eq!(
14195            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14196            0
14197        );
14198        assert!(
14199            runner
14200                .graft()
14201                .await
14202                .unwrap_err()
14203                .is::<crate::repo_lock::ResourceBlocked>()
14204        );
14205        assert!(matches!(
14206            runner.state.graft.as_ref().unwrap().outcome,
14207            GraftOutcome::Running
14208        ));
14209        assert_eq!(
14210            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14211            unfinished
14212        );
14213        assert!(leftover.exists());
14214        drop(external);
14215        std::fs::remove_file(&path).unwrap();
14216        std::fs::create_dir(&path).unwrap();
14217        assert!(runner.graft().await.is_err());
14218        assert!(matches!(
14219            runner.state.graft.as_ref().unwrap().outcome,
14220            GraftOutcome::Running
14221        ));
14222        assert_eq!(
14223            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14224            unfinished
14225        );
14226        assert!(leftover.exists());
14227        std::fs::remove_dir(&path).unwrap();
14228        runner.graft().await.unwrap();
14229        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14230        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
14231        assert!(!leftover.exists());
14232    }
14233
14234    #[tokio::test]
14235    #[cfg(unix)]
14236    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
14237        use std::os::fd::AsRawFd as _;
14238        let tmp = tempfile::tempdir().unwrap();
14239        let mut runner = registered_runner(tmp.path()).await;
14240        runner.state.prep_complete = Some(false);
14241        runner.state.config.graph.timeout_resource_wait = 1;
14242        runner.state.save().unwrap();
14243        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14244        assert_eq!(
14245            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14246            0
14247        );
14248        runner.execute().await.unwrap();
14249        let disk = RunState::load(&runner.state.id).unwrap();
14250        assert!(disk.driver_exited);
14251        assert_eq!(disk.driver_pid, None);
14252        assert_eq!(runner.state.status, RunStatus::Blocked);
14253        assert_eq!(
14254            runner.state.resource_block.as_ref().unwrap().kind,
14255            "repository coordination"
14256        );
14257        assert!(runner.state.prep_failure.is_none());
14258        assert!(runner.state.seats.is_empty());
14259        runner.pause.park();
14260        // Include headroom for Git subprocesses under parallel test load;
14261        // the repository contention deadline itself remains one second.
14262        tokio::time::timeout(Duration::from_secs(10), runner.execute())
14263            .await
14264            .unwrap()
14265            .unwrap();
14266        assert!(runner.state.parked);
14267        assert!(runner.state.prep_failure.is_none());
14268        let disk = RunState::load(&runner.state.id).unwrap();
14269        assert!(disk.driver_exited);
14270        assert_eq!(disk.driver_pid, None);
14271    }
14272
14273    #[tokio::test]
14274    #[cfg(unix)]
14275    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
14276        use std::os::fd::AsRawFd as _;
14277        let tmp = tempfile::tempdir().unwrap();
14278        let mut runner = registered_runner(tmp.path()).await;
14279        runner.state.prep_complete = Some(false);
14280        runner.state.config.graph.timeout_resource_wait = 30;
14281        runner.state.save().unwrap();
14282        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14283        assert_eq!(
14284            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14285            0
14286        );
14287        assert!(
14288            tokio::time::timeout(Duration::from_millis(100), runner.execute())
14289                .await
14290                .is_err()
14291        );
14292        let disk = RunState::load(&runner.state.id).unwrap();
14293        assert!(disk.driver_exited);
14294        assert_eq!(disk.driver_pid, None);
14295        assert!(disk.status.resumable());
14296        assert!(runner.state.driver_exited);
14297        assert_eq!(runner.state.driver_pid, None);
14298        runner.state.save().unwrap();
14299        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
14300    }
14301
14302    #[tokio::test]
14303    #[cfg(unix)]
14304    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
14305        let tmp = tempfile::tempdir().unwrap();
14306        let mut runner = registered_runner(tmp.path()).await;
14307        let winner = runner.state.candidates[0].clone();
14308        let cache = tmp.path().join("cache");
14309        let started = tmp.path().join("pre-gate-started");
14310        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
14311        runner.state.config.verify.gate =
14312            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
14313        let home = crate::run::home();
14314        let owner =
14315            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
14316        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
14317            crate::cache::AcquireOutcome::Acquired(g) => g,
14318            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
14319        };
14320        let mut gate = Box::pin(runner.run_gate(&winner));
14321        assert!(
14322            tokio::time::timeout(Duration::from_millis(150), &mut gate)
14323                .await
14324                .is_err()
14325        );
14326        assert!(!started.exists(), "pre-gate must wait for the cache too");
14327        // Simulate the wave that already owns the cache entering invoke.
14328        // It must obtain the editor reservation and finish without waiting
14329        // for the gate that is queued behind its cache.
14330        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
14331        let editor = crate::repo_lock::SeatLease::checkout_with(
14332            &winner.worktree,
14333            true,
14334            crate::config::test_coordination().as_ref(),
14335            &wait,
14336        )
14337        .await
14338        .unwrap();
14339        drop(editor);
14340        guard.release();
14341        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14342            .await
14343            .unwrap()
14344            .unwrap();
14345        drop(gate);
14346        assert!(outcomes.iter().all(CommandOutcome::ok));
14347        assert!(started.exists());
14348    }
14349
14350    #[tokio::test]
14351    #[cfg(unix)]
14352    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
14353        use std::os::fd::AsRawFd as _;
14354
14355        ask_test_home();
14356        let tmp = tempfile::tempdir().unwrap();
14357        let repo = tmp.path().join("repo");
14358        std::fs::create_dir_all(&repo).unwrap();
14359        init_repo(&repo);
14360        let common = repo.join(".git");
14361        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14362        let mut runner = runner_at(RunStatus::Prep);
14363        runner.state.repo = repo.clone();
14364        runner.state.config.coordination = crate::config::test_coordination();
14365        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14366        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14367        runner.state.config.graph.advise = false;
14368        runner.roles.implementers = vec![spec("implement")];
14369        runner.roles.judges = vec![spec("judge")];
14370        runner.prep().await.unwrap();
14371        let winner = runner.state.candidates[0].clone();
14372        // Emulate edits already left by a completed seat. The fixture CLI
14373        // only reports completion; it performs no repository operations.
14374        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
14375        let completed = tmp.path().join("seat-completed");
14376        runner.roles.implementers[0].command = vec![
14377            "sh".into(),
14378            "-c".into(),
14379            format!("touch '{}'; echo completed", completed.display()),
14380        ];
14381        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
14382        assert_eq!(
14383            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14384            0
14385        );
14386        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14387        let mut implement = Box::pin(runner.implement());
14388        tokio::select! {
14389            result = &mut implement => panic!("rescue must wait: {result:?}"),
14390            result = tokio::time::timeout(Duration::from_secs(10), async {
14391                while !completed.exists() {
14392                    tokio::time::sleep(Duration::from_millis(20)).await;
14393                }
14394            }) => result.unwrap(),
14395        }
14396        assert!(
14397            tokio::time::timeout(Duration::from_millis(200), &mut implement)
14398                .await
14399                .is_err()
14400        );
14401        assert_eq!(
14402            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14403            before
14404        );
14405        assert_eq!(
14406            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14407            0
14408        );
14409        tokio::time::timeout(Duration::from_secs(10), &mut implement)
14410            .await
14411            .unwrap()
14412            .unwrap();
14413        drop(implement);
14414        assert_ne!(
14415            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14416            before
14417        );
14418        assert!(git::is_clean(&winner.worktree).await.unwrap());
14419
14420        let started = tmp.path().join("pre-gate-started");
14421        let proceed = tmp.path().join("pre-gate-proceed");
14422        let checked = tmp.path().join("gate-ran");
14423        runner.state.config.verify.pre_gate = vec![format!(
14424            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14425            started.display(),
14426            proceed.display()
14427        )];
14428        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
14429        assert_eq!(
14430            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14431            0
14432        );
14433        let mut gate = Box::pin(runner.run_gate(&winner));
14434        assert!(
14435            tokio::time::timeout(Duration::from_millis(200), &mut gate)
14436                .await
14437                .is_err()
14438        );
14439        assert!(!started.exists());
14440        assert!(!checked.exists());
14441        assert!(git::is_clean(&winner.worktree).await.unwrap());
14442        assert_eq!(
14443            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14444            0
14445        );
14446        tokio::select! {
14447            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
14448            result = tokio::time::timeout(Duration::from_secs(10), async {
14449                while !started.exists() {
14450                    tokio::time::sleep(Duration::from_millis(20)).await;
14451                }
14452            }) => result.unwrap(),
14453        }
14454        assert_eq!(
14455            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14456            -1,
14457            "mechanical edits must hold the real lock"
14458        );
14459        std::fs::write(proceed, "ready").unwrap();
14460        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14461            .await
14462            .unwrap()
14463            .unwrap();
14464        drop(gate);
14465        assert!(outcomes.iter().all(CommandOutcome::ok));
14466        assert!(checked.exists());
14467        assert!(runner.state.pre_gate_commit.is_some());
14468        assert!(git::is_clean(&winner.worktree).await.unwrap());
14469        assert_eq!(
14470            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14471            0,
14472            "gate completion must release coordination"
14473        );
14474        drop(outsider);
14475        std::fs::remove_file(common.join("updater.lock")).unwrap();
14476        std::fs::create_dir(common.join("updater.lock")).unwrap();
14477        std::fs::remove_file(&started).unwrap();
14478        std::fs::remove_file(&checked).unwrap();
14479        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14480        let error = runner.run_gate(&winner).await.unwrap_err();
14481        assert!(
14482            error
14483                .to_string()
14484                .contains("acquire mandatory repository lock")
14485        );
14486        assert!(
14487            !started.exists(),
14488            "failed acquisition must prevent pre-gate"
14489        );
14490        assert!(!checked.exists(), "failed acquisition must prevent gate");
14491        assert!(git::is_clean(&winner.worktree).await.unwrap());
14492        assert_eq!(
14493            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14494            head
14495        );
14496    }
14497
14498    #[tokio::test]
14499    #[cfg(unix)]
14500    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
14501        use std::os::fd::AsRawFd as _;
14502        use std::time::Duration;
14503
14504        ask_test_home();
14505        let tmp = tempfile::tempdir().unwrap();
14506        let repo = tmp.path().join("repo");
14507        std::fs::create_dir_all(&repo).unwrap();
14508        init_repo(&repo);
14509        let common = repo.join(".git");
14510        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14511        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
14512        assert_eq!(
14513            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14514            0
14515        );
14516        let mut runner = runner_at(RunStatus::Prep);
14517        runner.state.repo = repo.clone();
14518        runner.state.config.coordination = crate::config::test_coordination();
14519        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14520        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14521        runner.state.config.graph.advise = true;
14522        runner.state.config.graph.advisors = 1;
14523        runner.roles.implementers = vec![spec("implement")];
14524        runner.roles.judges = vec![spec("judge")];
14525        let root = runner.state.worktree_root();
14526        let started = tmp.path().join("setup-started");
14527        let proceed = tmp.path().join("setup-proceed");
14528        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14529            run: Some(format!(
14530                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14531                started.display(),
14532                proceed.display()
14533            )),
14534            ..Default::default()
14535        }];
14536        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
14537        let config = git::git(&repo, &["config", "--local", "--list"])
14538            .await
14539            .unwrap();
14540        let mut prep = Box::pin(runner.prep());
14541        assert!(
14542            tokio::time::timeout(Duration::from_millis(200), &mut prep)
14543                .await
14544                .is_err()
14545        );
14546        assert!(
14547            !root.exists(),
14548            "contention must prevent even checkout creation"
14549        );
14550        assert!(!started.exists(), "contention must prevent setup");
14551        assert_eq!(
14552            git::git(&repo, &["branch", "--list"]).await.unwrap(),
14553            branches
14554        );
14555        assert_eq!(
14556            git::git(&repo, &["config", "--local", "--list"])
14557                .await
14558                .unwrap(),
14559            config
14560        );
14561        drop(lock);
14562        tokio::select! {
14563            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
14564            result = tokio::time::timeout(Duration::from_secs(10), async {
14565                while !started.exists() {
14566                    tokio::time::sleep(Duration::from_millis(20)).await;
14567                }
14568            }) => result.unwrap(),
14569        }
14570        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
14571        assert_eq!(
14572            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14573            -1
14574        );
14575        assert!(
14576            tokio::time::timeout(
14577                Duration::from_millis(100),
14578                crate::repo_lock::SeatLease::for_checkout(
14579                    &repo,
14580                    true,
14581                    crate::config::test_coordination().as_ref()
14582                )
14583            )
14584            .await
14585            .is_err(),
14586            "preparation must also exclude local editors"
14587        );
14588        std::fs::write(proceed, "ready").unwrap();
14589        tokio::time::timeout(Duration::from_secs(10), &mut prep)
14590            .await
14591            .unwrap()
14592            .unwrap();
14593        drop(prep);
14594        assert_eq!(runner.state.prep_complete, Some(true));
14595        assert!(root.join("judge-1").exists());
14596        assert!(root.join("advisor-1").exists());
14597        assert_eq!(
14598            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14599            0
14600        );
14601    }
14602
14603    #[tokio::test]
14604    #[cfg(unix)]
14605    async fn preparation_lock_failure_prevents_repository_changes() {
14606        ask_test_home();
14607        let tmp = tempfile::tempdir().unwrap();
14608        let repo = tmp.path().join("repo");
14609        std::fs::create_dir_all(&repo).unwrap();
14610        init_repo(&repo);
14611        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14612        // A directory at the required lock path makes opening the real lock
14613        // fail, rather than merely exercising an invalid registration.
14614        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
14615        let mut runner = runner_at(RunStatus::Prep);
14616        runner.state.repo = repo.clone();
14617        runner.state.config.coordination = crate::config::test_coordination();
14618        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14619        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14620        runner.roles.implementers = vec![spec("implement")];
14621        let error = runner.prep().await.unwrap_err();
14622        assert!(
14623            error
14624                .to_string()
14625                .contains("acquire mandatory repository lock")
14626        );
14627        assert!(!runner.state.worktree_root().exists());
14628        assert!(runner.state.candidates.is_empty());
14629        assert!(!repo.join(".git/config.worktree").exists());
14630        assert_eq!(
14631            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
14632            "* main"
14633        );
14634    }
14635
14636    #[tokio::test]
14637    async fn preparation_records_creation_intent_before_git_can_fail() {
14638        ask_test_home();
14639        let tmp = tempfile::tempdir().unwrap();
14640        let repo = tmp.path().join("repo");
14641        std::fs::create_dir_all(&repo).unwrap();
14642        init_repo(&repo);
14643        let mut runner = runner_at(RunStatus::Prep);
14644        runner.state.repo = repo.clone();
14645        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14646        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14647        runner.state.config.blind.commit_msg_hook = false;
14648        runner.roles.implementers = vec![spec("one")];
14649        let label = blind::assign_labels(1, runner.state.seed)[0];
14650        let branch = runner.state.branch_for(label);
14651        // A conflicting branch makes Git refuse creation before a candidate
14652        // can be registered. The durable intent must still identify it.
14653        git::git(&repo, &["branch", &branch]).await.unwrap();
14654        assert!(runner.prep().await.is_err());
14655        let saved = RunState::load(&runner.state.id).unwrap();
14656        assert!(saved.candidates.is_empty());
14657        let event = saved
14658            .events
14659            .iter()
14660            .find(|e| e.node == "prep-intent")
14661            .unwrap();
14662        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
14663        assert_eq!(intent["branch"], branch);
14664        assert_eq!(intent["start"], runner.state.base_commit);
14665        assert_eq!(
14666            intent["worktree"],
14667            runner
14668                .state
14669                .worktree_root()
14670                .join(format!("cand-{label}"))
14671                .to_string_lossy()
14672                .as_ref()
14673        );
14674        assert_eq!(intent["picks"], serde_json::json!([]));
14675    }
14676
14677    #[tokio::test]
14678    async fn invalid_pinned_sources_create_no_partial_worktrees() {
14679        ask_test_home();
14680        let tmp = tempfile::tempdir().unwrap();
14681        let repo = tmp.path().join("repo");
14682        std::fs::create_dir_all(&repo).unwrap();
14683        init_repo(&repo);
14684        let mut runner = runner_at(RunStatus::Prep);
14685        runner.state.repo = repo.clone();
14686        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14687        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14688        runner.state.sources.pick_commits.push(refs::Input {
14689            token: "missing".into(),
14690            sha: "0".repeat(40),
14691            provenance: refs::Provenance::ExplicitCli,
14692        });
14693        assert!(runner.prep().await.is_err());
14694        assert!(runner.state.candidates.is_empty());
14695        assert!(!runner.state.worktree_root().exists());
14696    }
14697
14698    #[tokio::test]
14699    async fn prose_and_legacy_inference_never_import_into_candidates() {
14700        ask_test_home();
14701        let tmp = tempfile::tempdir().unwrap();
14702        let repo = tmp.path().join("repo");
14703        std::fs::create_dir_all(&repo).unwrap();
14704        init_repo(&repo);
14705        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14706        git::git(&repo, &["checkout", "-b", "magi/history/A"])
14707            .await
14708            .unwrap();
14709        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
14710        git::git(&repo, &["add", "."]).await.unwrap();
14711        git::git(&repo, &["commit", "-m", "historical implementation"])
14712            .await
14713            .unwrap();
14714        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
14715        git::git(&repo, &["checkout", "main"]).await.unwrap();
14716        for instruction in [
14717            format!("Example: magi/history/A at {source}"),
14718            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
14719            format!("Never import magi/history/A or {source}"),
14720            format!("Conversation answer: I mentioned {source} only as background"),
14721        ] {
14722            let mut runner = runner_at(RunStatus::Prep);
14723            runner.state.repo = repo.clone();
14724            runner.state.base_commit = base.clone();
14725            runner.state.instruction = instruction.clone();
14726            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
14727            runner.state.config.blind.commit_msg_hook = false;
14728            runner.state.seeds = vec![refs::Seed {
14729                token: "magi/history/A".into(),
14730                sha: source.clone(),
14731                kind: refs::SeedKind::Unmerged,
14732                branch: true,
14733                detail: String::new(),
14734            }];
14735            runner.roles.implementers = vec![spec("one")];
14736            assert_eq!(seeded_instruction(&runner.state), instruction);
14737            // Model an old record with inferred seeds but no structured input field.
14738            let mut legacy = serde_json::to_value(&runner.state).unwrap();
14739            legacy.as_object_mut().unwrap().remove("sources");
14740            runner.state = serde_json::from_value(legacy).unwrap();
14741            assert_eq!(runner.state.sources, refs::Sources::default());
14742            runner.state.save().unwrap();
14743            runner.state = RunState::load(&runner.state.id).unwrap();
14744            runner.prep().await.unwrap();
14745            for candidate in &runner.state.candidates {
14746                assert_eq!(
14747                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
14748                    base
14749                );
14750                assert!(!candidate.worktree.join("unwanted").exists());
14751            }
14752        }
14753    }
14754
14755    #[tokio::test]
14756    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
14757        ask_test_home();
14758        let tmp = tempfile::tempdir().unwrap();
14759        let repo = tmp.path().join("repo");
14760        std::fs::create_dir_all(&repo).unwrap();
14761        init_repo(&repo);
14762        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14763        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
14764        std::fs::write(repo.join("referenced"), "required commit").unwrap();
14765        git::git(&repo, &["add", "."]).await.unwrap();
14766        git::git(&repo, &["commit", "-m", "required seed"])
14767            .await
14768            .unwrap();
14769        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
14770        git::git(&repo, &["checkout", "main"]).await.unwrap();
14771        let mut runner = runner_at(RunStatus::Prep);
14772        runner.state.repo = repo.clone();
14773        runner.state.base_commit = base.clone();
14774        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14775        runner.state.config.graph.advise = false;
14776        runner.state.config.blind.commit_msg_hook = false;
14777        runner.state.sources =
14778            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
14779                .await
14780                .unwrap();
14781        runner.roles.implementers = vec![spec("one"), spec("two")];
14782        runner.roles.judges = vec![spec("judge")];
14783        let label = blind::assign_labels(2, runner.state.seed)[0];
14784        let branch = runner.state.branch_for(label);
14785        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14786        git::worktree_add_branch(&repo, &worktree, &branch, &base)
14787            .await
14788            .unwrap();
14789        runner.state.candidates.push(Candidate {
14790            index: 0,
14791            label,
14792            agent: "one".into(),
14793            branch,
14794            worktree,
14795            summary: String::new(),
14796            stat: String::new(),
14797            files: 0,
14798            commits: 0,
14799            empty: false,
14800            failed: None,
14801            verified_noop: None,
14802            duration_ms: 0,
14803            folded: false,
14804        });
14805        runner.state.save().unwrap();
14806        // Reload the checkpoint immediately before its first cherry-pick.
14807        runner.state = RunState::load(&runner.state.id).unwrap();
14808        assert_eq!(runner.state.prep_complete, Some(false));
14809        runner.prep().await.unwrap();
14810        assert_eq!(runner.state.prep_complete, Some(true));
14811        assert_eq!(runner.state.candidates.len(), 2);
14812        for candidate in &runner.state.candidates {
14813            assert_eq!(
14814                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14815                git::tree_of(&repo, &seed).await.unwrap()
14816            );
14817            assert_eq!(
14818                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14819                "required commit"
14820            );
14821        }
14822        assert!(runner.state.worktree_root().join("judge-1").exists());
14823        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14824        runner.prep().await.unwrap();
14825        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14826
14827        runner.state.prep_complete = Some(false);
14828        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14829            run: Some(format!(
14830                "test -f '{}'",
14831                tmp.path().join("dependency-ready").display()
14832            )),
14833            ..Default::default()
14834        }];
14835        assert!(runner.prep().await.is_err());
14836        assert!(runner.state.prep_failure.is_some());
14837        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14838        runner.state = RunState::load(&runner.state.id).unwrap();
14839        assert!(
14840            runner.prep().await.is_err(),
14841            "correcting the cause alone must not auto-retry"
14842        );
14843        let failed_at = runner.state.events.last().unwrap().at;
14844        assert!(
14845            runner.retry_preparation(failed_at).is_err(),
14846            "old approvals cannot unlock the hold"
14847        );
14848        let mut task = crate::queue::Task::new(
14849            "retry".into(),
14850            "retry setup".into(),
14851            repo.clone(),
14852            crate::queue::Source::Human,
14853        );
14854        task.start(runner.state.id.clone());
14855        task.hold_machine(Some("setup failed".into()));
14856        task.release_operator("cli", None);
14857        runner
14858            .retry_preparation(task.resumption.as_ref().unwrap().at)
14859            .unwrap();
14860        runner.prep().await.unwrap();
14861        assert!(runner.state.prep_failure.is_none());
14862        assert_eq!(runner.state.prep_complete, Some(true));
14863        assert_eq!(runner.state.candidates.len(), 2);
14864        assert!(runner.state.worktree_root().join("judge-1").exists());
14865        for candidate in &runner.state.candidates {
14866            assert_eq!(
14867                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14868                git::tree_of(&repo, &seed).await.unwrap()
14869            );
14870        }
14871    }
14872
14873    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14874        let mut heads = Vec::new();
14875        for candidate in candidates {
14876            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14877        }
14878        heads
14879    }
14880
14881    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14882        ask_test_home();
14883        let tmp = tempfile::tempdir().unwrap();
14884        let repo = tmp.path().join("repo");
14885        std::fs::create_dir(&repo).unwrap();
14886        init_repo(&repo);
14887        let mut runner = runner_at(RunStatus::Reviewing);
14888        runner.state.repo = repo.clone();
14889        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14890        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14891        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14892            let branch = format!("candidate-{label}");
14893            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14894            git::worktree_add_branch(&repo, &worktree, &branch, "main")
14895                .await
14896                .unwrap();
14897            if label == 'B' {
14898                std::fs::write(worktree.join("change"), "winner").unwrap();
14899                git::rescue_commit(&worktree, "candidate change")
14900                    .await
14901                    .unwrap();
14902            }
14903            runner.state.candidates.push(Candidate {
14904                index,
14905                label,
14906                agent: "alpha".into(),
14907                branch,
14908                worktree,
14909                summary: "completed reply".into(),
14910                stat: String::new(),
14911                files: usize::from(label == 'B'),
14912                commits: usize::from(label == 'B'),
14913                empty: label == 'A',
14914                failed: None,
14915                verified_noop: None,
14916                duration_ms: 1,
14917                folded: false,
14918            });
14919        }
14920        runner.state.judge_skipped = true;
14921        runner.state.tally = Some(Tally {
14922            first_choice: BTreeMap::new(),
14923            borda: BTreeMap::new(),
14924            winner: 'B',
14925            rankings: 0,
14926            unanimous_initial: false,
14927            deliberated: false,
14928            changed_votes: 0,
14929            unanimous_final: false,
14930            tie_break: None,
14931            judges: 0,
14932            present: 0,
14933            quorum: 0,
14934            met_quorum: true,
14935            uncontested: Some("only B produced a change".into()),
14936        });
14937        runner.state.save().unwrap();
14938        (tmp, runner)
14939    }
14940
14941    #[tokio::test]
14942    #[cfg(unix)]
14943    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14944        for (solo, failed_checkpoint) in
14945            [(true, false), (false, false), (true, true), (false, true)]
14946        {
14947            let (_tmp, mut runner) = preservation_resume_fixture().await;
14948            if solo {
14949                runner.state.candidates.retain(|c| c.label == 'A');
14950            }
14951            runner.state.candidates[0].empty = false;
14952            runner.state.candidates[0].summary.clear();
14953            runner.state.tally = None;
14954            runner.state.judge_skipped = false;
14955            runner.state.status = RunStatus::Implementing;
14956            runner.state.config.disk.min_free_bytes = u64::MAX;
14957            let mut implementer = spec("alpha");
14958            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14959            runner.roles.implementers = vec![implementer];
14960            let patch = runner.state.dir().join("artifacts/cand-A.patch");
14961            if failed_checkpoint {
14962                std::fs::create_dir_all(&patch).unwrap();
14963            }
14964            runner.state.save().unwrap();
14965            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14966            runner.state = RunState::load(&runner.state.id).unwrap();
14967            assert_eq!(runner.state.status, RunStatus::Blocked);
14968            assert!(runner.state.interrupted_implementations.contains(&'A'));
14969            assert_eq!(runner.state.candidates[0].commits, 0);
14970            assert!(!runner.state.candidates[0].empty);
14971            assert!(runner.state.candidates[0].failed.is_none());
14972            if failed_checkpoint {
14973                assert!(runner.state.candidate_preservation.contains_key(&'A'));
14974                std::fs::remove_dir(patch).unwrap();
14975            }
14976            runner.state.resource_block = None;
14977            runner.state.config.disk.min_free_bytes = 0;
14978            runner.roles.implementers[0].command = vec![
14979                "sh".into(),
14980                "-c".into(),
14981                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14982            ];
14983            runner.implement().await.unwrap();
14984            assert!(runner.state.interrupted_implementations.is_empty());
14985            assert!(runner.state.candidate_preservation.is_empty());
14986            assert_eq!(runner.state.candidates[0].commits, 1);
14987            assert!(!runner.state.candidates[0].empty);
14988            assert!(
14989                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14990                    .unwrap()
14991                    .contains("retried empty interrupted seat")
14992            );
14993            if !solo {
14994                assert_eq!(runner.state.candidates[1].commits, 1);
14995            }
14996        }
14997    }
14998
14999    #[tokio::test]
15000    #[cfg(unix)]
15001    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15002        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15003            let (_tmp, mut runner) = preservation_resume_fixture().await;
15004            runner.state.candidates.retain(|c| c.label == 'A');
15005            runner.state.candidates[0].empty = false;
15006            runner.state.candidates[0].summary.clear();
15007            runner.state.tally = None;
15008            runner.state.judge_skipped = false;
15009            runner.state.status = RunStatus::Implementing;
15010            // Force the real post-spawn watchdog, independently of available
15011            // disk space. Dirty/committed work predates its first measurement.
15012            runner.state.config.disk.min_free_bytes = u64::MAX;
15013            let wt = runner.state.candidates[0].worktree.clone();
15014            std::fs::write(
15015                wt.join("interrupted-source"),
15016                "retain implementation work\n",
15017            )
15018            .unwrap();
15019            if committed {
15020                git::rescue_commit(&wt, "implementation checkpoint")
15021                    .await
15022                    .unwrap();
15023            }
15024            let lock = git::git(
15025                &wt,
15026                &[
15027                    "rev-parse",
15028                    "--path-format=absolute",
15029                    "--git-path",
15030                    "index.lock",
15031                ],
15032            )
15033            .await
15034            .unwrap();
15035            if block_rescue {
15036                std::fs::write(&lock, "fixture-owned lock").unwrap();
15037            }
15038            let mut implementer = spec("alpha");
15039            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15040            runner.roles.implementers = vec![implementer];
15041            runner.state.save().unwrap();
15042            let result = runner.implement().await;
15043            assert_eq!(result.is_err(), block_rescue);
15044            assert_eq!(
15045                runner.state.resource_block.as_ref().unwrap().kind,
15046                "disk pressure"
15047            );
15048            assert!(
15049                runner
15050                    .state
15051                    .events
15052                    .iter()
15053                    .any(|e| e.message.contains("agent execution interrupted"))
15054            );
15055            assert!(
15056                runner
15057                    .state
15058                    .dir()
15059                    .join("artifacts/candidate-A-recovery/staged.patch")
15060                    .exists()
15061            );
15062            if block_rescue {
15063                let saved = RunState::load(&runner.state.id).unwrap();
15064                assert!(saved.candidate_preservation.contains_key(&'A'));
15065                assert!(!saved.candidates[0].empty);
15066                assert_eq!(
15067                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15068                    "retain implementation work\n"
15069                );
15070                std::fs::remove_file(lock).unwrap();
15071                runner.state = saved;
15072            } else {
15073                assert_eq!(runner.state.status, RunStatus::Blocked);
15074                assert_eq!(runner.state.candidates[0].commits, 1);
15075            }
15076            // Recovery must save the candidate before deciding whether another
15077            // implementation is needed. No implementer is available on retry.
15078            runner.roles.implementers.clear();
15079            runner.state.config.disk.min_free_bytes = 0;
15080            runner.state.resource_block = None;
15081            runner.implement().await.unwrap();
15082            assert!(runner.state.candidate_preservation.is_empty());
15083            assert_eq!(runner.state.candidates[0].commits, 1);
15084            assert!(!runner.state.candidates[0].empty);
15085            assert!(git::is_clean(&wt).await.unwrap());
15086            assert!(
15087                crate::run::read_artifact(&runner.state, "cand-A.patch")
15088                    .unwrap()
15089                    .contains("retain implementation work")
15090            );
15091        }
15092    }
15093
15094    #[tokio::test]
15095    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15096        let (_tmp, mut runner) = preservation_resume_fixture().await;
15097        // Schema 15 already confirmed the unfolded no-change result.
15098        runner.implement().await.unwrap();
15099        assert!(runner.state.candidate_preservation.is_empty());
15100        assert!(
15101            !runner
15102                .state
15103                .dir()
15104                .join("artifacts/candidate-A-recovery")
15105                .exists()
15106        );
15107        runner.fold_losers().await.unwrap();
15108        assert!(!runner.state.candidates[0].worktree.exists());
15109        for schema in [14, crate::run::SCHEMA] {
15110            runner.state.schema = schema;
15111            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15112            runner.state.save().unwrap();
15113            runner.state = RunState::load(&runner.state.id).unwrap();
15114            runner.implement().await.unwrap();
15115            assert!(runner.state.candidate_preservation.is_empty());
15116            assert!(runner.state.candidates[0].folded);
15117            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15118        }
15119    }
15120
15121    #[tokio::test]
15122    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15123        let (_tmp, mut runner) = preservation_resume_fixture().await;
15124        runner.state.candidates.retain(|c| c.label == 'A');
15125        runner.state.tally = None;
15126        runner.state.judge_skipped = false;
15127        let candidate = &mut runner.state.candidates[0];
15128        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15129        candidate.failed = Some("deployment validation unavailable".into());
15130        std::fs::write(
15131            candidate.worktree.join("repair"),
15132            "future launcher policy\n",
15133        )
15134        .unwrap();
15135        runner
15136            .state
15137            .candidate_preservation
15138            .insert('A', "awaiting persistence".into());
15139        runner.persist_candidates().await.unwrap();
15140        let candidate = &runner.state.candidates[0];
15141        assert!(!candidate.empty);
15142        assert_eq!(candidate.commits, 1);
15143        assert_eq!(candidate.files, 1);
15144        assert!(candidate.failed.is_none());
15145        assert!(candidate.summary.contains("EPERM"));
15146        assert!(
15147            crate::run::read_artifact(&runner.state, "cand-A.patch")
15148                .unwrap()
15149                .contains("future launcher policy")
15150        );
15151        assert!(runner.state.candidate_preservation.is_empty());
15152    }
15153
15154    #[tokio::test]
15155    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
15156        let (_tmp, mut runner) = preservation_resume_fixture().await;
15157        let wt = runner.state.candidates[0].worktree.clone();
15158        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
15159        runner.state.schema = 14;
15160        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15161        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15162        let tally = serde_json::to_value(&runner.state.tally).unwrap();
15163        for _ in 0..2 {
15164            assert!(runner.implement().await.is_err());
15165            assert_eq!(runner.state.status, RunStatus::Blocked);
15166            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15167            assert!(
15168                runner.state.candidates[0].empty,
15169                "old verdict must remain unusable"
15170            );
15171            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
15172            assert!(runner.state.judge_skipped);
15173            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
15174            assert!(
15175                crate::run::read_artifact(&runner.state, "cand-A.patch")
15176                    .unwrap()
15177                    .contains("owned edits")
15178            );
15179            assert!(wt.exists());
15180            assert_eq!(
15181                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
15182                    .await
15183                    .unwrap(),
15184                1
15185            );
15186            runner.state = RunState::load(&runner.state.id).unwrap();
15187        }
15188    }
15189
15190    #[tokio::test]
15191    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
15192        let (_tmp, mut runner) = preservation_resume_fixture().await;
15193        let wt = runner.state.candidates[0].worktree.clone();
15194        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
15195        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
15196        let index_lock = PathBuf::from(
15197            git::git(
15198                &wt,
15199                &[
15200                    "rev-parse",
15201                    "--path-format=absolute",
15202                    "--git-path",
15203                    "index.lock",
15204                ],
15205            )
15206            .await
15207            .unwrap(),
15208        );
15209        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15210        runner.state.schema = 14;
15211        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15212        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15213        assert!(runner.implement().await.is_err());
15214        assert!(runner.after_implement().is_err());
15215        assert!(runner.fold_losers().await.is_err());
15216        assert_eq!(
15217            std::fs::read_to_string(wt.join("recovered")).unwrap(),
15218            "failed seat edits"
15219        );
15220        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15221        runner.state = RunState::load(&runner.state.id).unwrap();
15222        std::fs::remove_file(index_lock).unwrap();
15223        // Recovery finds changes that the old uncontested result never judged.
15224        assert!(runner.implement().await.is_err());
15225        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15226        assert!(runner.state.candidates[0].empty);
15227        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15228        assert!(
15229            crate::run::read_artifact(&runner.state, "cand-A.patch")
15230                .unwrap()
15231                .contains("failed seat edits")
15232        );
15233        assert!(wt.exists());
15234    }
15235
15236    #[tokio::test]
15237    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
15238        let (_tmp, mut runner) = preservation_resume_fixture().await;
15239        let repo = runner.state.repo.clone();
15240        let common = git::git(
15241            &repo,
15242            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
15243        )
15244        .await
15245        .unwrap();
15246        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
15247        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
15248        assert!(runner.fold_losers().await.is_err());
15249        assert!(!runner.state.candidates[0].worktree.exists());
15250        assert!(!runner.state.candidates[0].folded);
15251        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
15252        runner.state = RunState::load(&runner.state.id).unwrap();
15253        std::fs::remove_file(branch_lock).unwrap();
15254        let judge = runner.state.worktree_root().join("judge-1");
15255        git::worktree_add_detached(&repo, &judge, "main")
15256            .await
15257            .unwrap();
15258        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
15259        runner.roles.judges = vec![spec("judge")];
15260        assert!(runner.fold_losers().await.is_err());
15261        runner.state = RunState::load(&runner.state.id).unwrap();
15262        assert!(
15263            runner.state.candidates[0].folded,
15264            "candidate progress saved before judge failure"
15265        );
15266        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
15267        assert!(judge.join("uncommitted").exists());
15268        std::fs::remove_file(judge.join("uncommitted")).unwrap();
15269        runner.fold_losers().await.unwrap();
15270        assert!(!judge.exists());
15271        runner.fold_losers().await.unwrap();
15272    }
15273
15274    #[tokio::test]
15275    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
15276        let (_tmp, mut runner) = preservation_resume_fixture().await;
15277        runner.state.candidates.retain(|c| c.label == 'B');
15278        runner.state.tally = None;
15279        runner.state.judge_skipped = false;
15280        runner.state.prep_complete = Some(true);
15281        runner.state.config.graph.advise = false;
15282        runner.state.config.graph.selected_implementer = Some("alpha".into());
15283        let wt = runner.state.candidates[0].worktree.clone();
15284        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
15285        runner
15286            .state
15287            .candidate_preservation
15288            .insert('B', "awaiting persistence".into());
15289        let index_lock = git::git(
15290            &wt,
15291            &[
15292                "rev-parse",
15293                "--path-format=absolute",
15294                "--git-path",
15295                "index.lock",
15296            ],
15297        )
15298        .await
15299        .unwrap();
15300        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15301        runner.state.save().unwrap();
15302        assert!(runner.execute_graph().await.is_err());
15303        assert_eq!(runner.state.status, RunStatus::Blocked);
15304        assert!(runner.state.candidate_preservation.contains_key(&'B'));
15305        std::fs::remove_file(index_lock).unwrap();
15306        // Park at the next graph boundary: this tests the real selected-seat
15307        // hold guard without dispatching reviewers or publishing anything.
15308        for _ in 0..2 {
15309            runner.state = RunState::load(&runner.state.id).unwrap();
15310            runner.pause.park();
15311            runner.execute_graph().await.unwrap();
15312            assert_eq!(runner.state.status, RunStatus::Implementing);
15313            assert!(runner.state.candidate_preservation.is_empty());
15314            assert_eq!(runner.state.candidates[0].commits, 2);
15315            assert!(runner.state.tally.is_none());
15316        }
15317        runner.implement().await.unwrap();
15318        assert_eq!(runner.state.status, RunStatus::Judging);
15319        // A genuine selected-implementer hold must still require release.
15320        runner.state.status = RunStatus::Blocked;
15321        runner.state.candidates[0].failed = Some("implementation failed".into());
15322        runner.state.save().unwrap();
15323        let error = runner.execute_graph().await.unwrap_err();
15324        assert!(
15325            error
15326                .to_string()
15327                .contains("selected implementation is held")
15328        );
15329    }
15330
15331    #[tokio::test]
15332    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
15333        let (_tmp, mut runner) = preservation_resume_fixture().await;
15334        let repo = runner.state.repo.clone();
15335        let candidate = runner.state.candidates[0].clone();
15336        // Remove only the disposable fixture directory, leaving Git's linked
15337        // worktree registration and branch intact, as in a vanished checkout.
15338        let registered = git::worktree_holding(&repo, &candidate.branch)
15339            .await
15340            .unwrap()
15341            .unwrap();
15342        assert_eq!(
15343            registered.canonicalize().unwrap(),
15344            candidate.worktree.canonicalize().unwrap()
15345        );
15346        std::fs::remove_dir_all(&candidate.worktree).unwrap();
15347        assert_eq!(
15348            git::worktree_holding(&repo, &candidate.branch)
15349                .await
15350                .unwrap(),
15351            Some(registered)
15352        );
15353        assert!(
15354            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
15355            "the stale registration initially keeps the branch checked out"
15356        );
15357        runner.fold_losers().await.unwrap();
15358        assert!(runner.state.candidates[0].folded);
15359        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15360        assert_eq!(
15361            git::worktree_holding(&repo, &candidate.branch)
15362                .await
15363                .unwrap(),
15364            None
15365        );
15366        runner.state = RunState::load(&runner.state.id).unwrap();
15367        runner.fold_losers().await.unwrap();
15368    }
15369
15370    #[tokio::test]
15371    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
15372        let (_tmp, mut runner) = preservation_resume_fixture().await;
15373        let repo = runner.state.repo.clone();
15374        let candidate = runner.state.candidates[0].clone();
15375        let winner = runner.state.candidates[1].clone();
15376        let registered = git::worktree_holding(&repo, &candidate.branch)
15377            .await
15378            .unwrap()
15379            .unwrap();
15380        assert_eq!(
15381            registered.canonicalize().unwrap(),
15382            candidate.worktree.canonicalize().unwrap()
15383        );
15384        std::fs::remove_dir_all(&candidate.worktree).unwrap();
15385        assert_eq!(
15386            git::worktree_holding(&repo, &candidate.branch)
15387                .await
15388                .unwrap(),
15389            Some(registered)
15390        );
15391        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
15392
15393        for _ in 0..2 {
15394            fold_run(&mut runner.state, false, &crate::run::home())
15395                .await
15396                .unwrap();
15397            runner.state = RunState::load(&runner.state.id).unwrap();
15398            assert!(runner.state.candidates[0].folded);
15399            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15400            assert_eq!(
15401                git::worktree_holding(&repo, &candidate.branch)
15402                    .await
15403                    .unwrap(),
15404                None
15405            );
15406            assert!(!runner.state.candidates[1].folded);
15407            assert!(winner.worktree.exists());
15408            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
15409        }
15410    }
15411
15412    #[tokio::test]
15413    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
15414        ask_test_home();
15415        let tmp = tempfile::tempdir().unwrap();
15416        let repo = tmp.path().join("repo");
15417        std::fs::create_dir(&repo).unwrap();
15418        init_repo(&repo);
15419        let mut runner = runner_at(RunStatus::Implementing);
15420        runner.state.repo = repo.clone();
15421        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15422        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15423        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15424            let branch = format!("candidate-{label}");
15425            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
15426            git::worktree_add_branch(&repo, &wt, &branch, "main")
15427                .await
15428                .unwrap();
15429            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
15430            git::git(&wt, &["add", "README.md"]).await.unwrap();
15431            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
15432            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
15433            runner.state.candidates.push(Candidate {
15434                index,
15435                label,
15436                agent: "alpha".to_owned(),
15437                branch,
15438                worktree: wt,
15439                summary: "completed implementation".to_owned(),
15440                stat: String::new(),
15441                files: 0,
15442                commits: 0,
15443                empty: false,
15444                failed: None,
15445                verified_noop: None,
15446                duration_ms: 42,
15447                folded: false,
15448            });
15449            runner
15450                .state
15451                .candidate_preservation
15452                .insert(label, "awaiting persistence".to_owned());
15453        }
15454        let wt = runner.state.candidates[0].worktree.clone();
15455        let index_lock = git::git(
15456            &wt,
15457            &[
15458                "rev-parse",
15459                "--path-format=absolute",
15460                "--git-path",
15461                "index.lock",
15462            ],
15463        )
15464        .await
15465        .unwrap();
15466        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15467        runner.state.save().unwrap();
15468        assert!(runner.persist_candidates().await.is_err());
15469        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15470        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
15471        assert_eq!(
15472            runner.state.candidates[1].commits, 1,
15473            "other results are still collected"
15474        );
15475        assert!(
15476            !runner.state.candidates[0].empty,
15477            "failed rescue is not no change"
15478        );
15479        assert_eq!(
15480            runner.state.candidates[0].summary,
15481            "completed implementation"
15482        );
15483        assert!(runner.after_implement().is_err());
15484        let home = tmp.path().join("fold-home");
15485        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
15486        assert!(wt.join("new-file").exists());
15487        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
15488        assert!(recovery.join("staged.patch").exists());
15489        assert!(recovery.join("unstaged.patch").exists());
15490        assert_eq!(
15491            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
15492            "untracked edits\n"
15493        );
15494        runner.state.begin_pass();
15495        runner.state.end_pass_parked(ParkCause::Upgrade {
15496            from: "1.0".into(),
15497            to: Some("1.1".into()),
15498        });
15499        runner.state.save().unwrap();
15500        runner.state = RunState::load(&runner.state.id).unwrap();
15501        assert!(matches!(
15502            runner.state.passes.last().unwrap().end,
15503            Some(crate::run::PassEnd::Parked {
15504                cause: ParkCause::Upgrade { .. }
15505            })
15506        ));
15507        std::fs::remove_file(index_lock).unwrap();
15508        // No implementer is configured: a persistence retry cannot dispatch a seat.
15509        runner.implement().await.unwrap();
15510        assert!(runner.state.candidate_preservation.is_empty());
15511        assert!(
15512            runner
15513                .state
15514                .candidates
15515                .iter()
15516                .all(|c| c.commits == 1 && !c.empty)
15517        );
15518        assert!(git::is_clean(&wt).await.unwrap());
15519        assert!(
15520            crate::run::read_artifact(&runner.state, "cand-A.patch")
15521                .unwrap()
15522                .contains("later tracked edits")
15523        );
15524    }
15525
15526    #[tokio::test]
15527    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
15528        ask_test_home();
15529        let tmp = tempfile::tempdir().unwrap();
15530        let repo = tmp.path().join("repo");
15531        std::fs::create_dir(&repo).unwrap();
15532        init_repo(&repo);
15533        let wt = tmp.path().join("candidate");
15534        git::worktree_add_branch(&repo, &wt, "candidate", "main")
15535            .await
15536            .unwrap();
15537        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15538        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
15539        assert!(wt.join("uncommitted").exists());
15540        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15541        std::fs::remove_file(wt.join("uncommitted")).unwrap();
15542        let mut runner = runner_at(RunStatus::Implementing);
15543        runner.state.repo = repo.clone();
15544        runner.state.base_commit = "missing-base".to_owned();
15545        runner.state.candidates.push(Candidate {
15546            index: 0,
15547            label: 'A',
15548            agent: "alpha".into(),
15549            branch: "candidate".into(),
15550            worktree: wt.clone(),
15551            summary: "completed".into(),
15552            stat: String::new(),
15553            files: 0,
15554            commits: 0,
15555            empty: false,
15556            failed: None,
15557            verified_noop: None,
15558            duration_ms: 1,
15559            folded: false,
15560        });
15561        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15562        assert!(
15563            fold_run(&mut runner.state, true, &tmp.path().join("home"))
15564                .await
15565                .is_err()
15566        );
15567        assert!(!runner.state.candidates[0].folded);
15568        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15569        std::fs::remove_file(wt.join("uncommitted")).unwrap();
15570        runner
15571            .state
15572            .candidate_preservation
15573            .insert('A', "pending".into());
15574        assert!(runner.persist_candidates().await.is_err());
15575        assert!(!runner.state.candidates[0].empty);
15576        assert!(!runner.state.candidates[0].folded);
15577        assert!(wt.exists());
15578        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15579        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
15580        std::fs::create_dir_all(&patch_path).unwrap();
15581        assert!(
15582            runner.persist_candidates().await.is_err(),
15583            "artifact failure stays pending"
15584        );
15585        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15586        assert!(!runner.state.candidates[0].empty);
15587        std::fs::remove_dir(patch_path).unwrap();
15588        runner.persist_candidates().await.unwrap();
15589        assert!(
15590            runner.state.candidates[0].empty,
15591            "only successful inspection confirms no change"
15592        );
15593    }
15594
15595    fn init_repo(dir: &Path) {
15596        let run = |args: &[&str]| {
15597            let out = std::process::Command::new("git")
15598                .args(args)
15599                .current_dir(dir)
15600                .quiet()
15601                .output()
15602                .expect("spawn git");
15603            assert!(
15604                out.status.success(),
15605                "git {args:?} failed: {}",
15606                String::from_utf8_lossy(&out.stderr)
15607            );
15608        };
15609        run(&["init", "-b", "main"]);
15610        run(&["config", "user.name", "magi test"]);
15611        run(&["config", "user.email", "magi@example.com"]);
15612        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
15613        run(&["add", "-A"]);
15614        run(&["commit", "-m", "init"]);
15615    }
15616
15617    // `settle_questions` is what closes the ghost the phone showed: a run's
15618    // seat asked something, the run then ended, and nothing was left to
15619    // abandon the question it left `open`. `HOME` is a process-wide
15620    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
15621    // first time it runs in the binary — every test below still reaches the
15622    // same directory whichever call won, and each gets its own run id from
15623    // `RunState::new`, so they never collide there.
15624    fn ask_test_home() {
15625        crate::run::pin_test_home();
15626    }
15627
15628    #[cfg(unix)]
15629    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
15630        ask_test_home();
15631        let tmp = tempfile::tempdir().unwrap();
15632        let repo = tmp.path().join("repo");
15633        std::fs::create_dir(&repo).unwrap();
15634        init_repo(&repo);
15635        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15636        let origin = tmp.path().join("origin.git");
15637        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
15638            .await
15639            .unwrap();
15640        git::git(
15641            &repo,
15642            &["remote", "add", "origin", origin.to_str().unwrap()],
15643        )
15644        .await
15645        .unwrap();
15646        git::git(&repo, &["checkout", "-b", "fix/retained"])
15647            .await
15648            .unwrap();
15649        std::fs::write(repo.join("retained"), "retained change").unwrap();
15650        git::git(&repo, &["add", "."]).await.unwrap();
15651        git::git(&repo, &["commit", "-m", "retained implementation"])
15652            .await
15653            .unwrap();
15654        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15655        git::git(&repo, &["checkout", "main"]).await.unwrap();
15656        std::fs::write(repo.join("base-change"), "new base").unwrap();
15657        git::git(&repo, &["add", "."]).await.unwrap();
15658        git::git(&repo, &["commit", "-m", "base moved"])
15659            .await
15660            .unwrap();
15661        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15662        git::git(&repo, &["checkout", "fix/retained"])
15663            .await
15664            .unwrap();
15665        let mut runner = runner_at(RunStatus::Voting);
15666        runner.state.repo = repo.clone();
15667        runner.state.base_commit = base;
15668        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15669        runner.state.config.disk.min_free_bytes = 0;
15670        runner.state.config.graph.review_rounds = 1;
15671        runner.state.config.verify.e2e = vec!["true".into()];
15672        runner.state.config.verify.gate = vec!["true".into()];
15673        runner.state.candidates = vec![Candidate {
15674            index: 0,
15675            label: 'A',
15676            agent: "fixture".into(),
15677            branch: "fix/retained".into(),
15678            worktree: repo.clone(),
15679            summary: String::new(),
15680            stat: String::new(),
15681            files: 1,
15682            commits: 1,
15683            empty: false,
15684            failed: None,
15685            verified_noop: None,
15686            duration_ms: 0,
15687            folded: false,
15688        }];
15689        runner.state.tally = Some(Tally {
15690            first_choice: BTreeMap::new(),
15691            borda: BTreeMap::new(),
15692            winner: 'A',
15693            rankings: 0,
15694            unanimous_initial: true,
15695            deliberated: false,
15696            changed_votes: 0,
15697            unanimous_final: true,
15698            tie_break: None,
15699            judges: 0,
15700            present: 0,
15701            quorum: 0,
15702            met_quorum: true,
15703            uncontested: Some("review-only".into()),
15704        });
15705        let mut task = crate::queue::Task::new(
15706            "fixture".into(),
15707            "fixture".into(),
15708            repo,
15709            crate::queue::Source::Human,
15710        );
15711        task.link_run(&runner.state.id);
15712        crate::queue::Queue::open().put(&mut task).unwrap();
15713        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
15714        runner.state.base_sync = Some(BaseSync {
15715            tip: String::new(),
15716            behind: 1,
15717            attempts: BASE_SYNC_ROUNDS,
15718            conflict: None,
15719            already_in: None,
15720        });
15721        runner.sync_to_base().await.unwrap();
15722        assert_eq!(runner.state.status, RunStatus::Blocked);
15723        (tmp, runner, head, task.id)
15724    }
15725
15726    #[cfg(unix)]
15727    #[tokio::test]
15728    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
15729        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
15730        if std::env::var_os(CHILD).is_none() {
15731            let home = tempfile::tempdir().unwrap();
15732            let output = std::process::Command::new(std::env::current_exe().unwrap())
15733                .args([
15734                    "--exact",
15735                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
15736                    "--nocapture",
15737                ])
15738                .env(CHILD, "1")
15739                .env("MAGI_HOME", home.path())
15740                .env("MAGI_WORKER_HOME", home.path())
15741                .env("MAGI_WORKER_ID", "abcdef")
15742                .output()
15743                .unwrap();
15744            assert!(
15745                output.status.success()
15746                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15747                "{}\n{}",
15748                String::from_utf8_lossy(&output.stdout),
15749                String::from_utf8_lossy(&output.stderr)
15750            );
15751            return;
15752        }
15753        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
15754        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15755        let home = crate::run::home();
15756        let queue = crate::queue::Queue::open();
15757        let mut owner = queue.get(&task).unwrap();
15758        owner.release_operator("test-release", None);
15759        owner.start(runner.state.id.clone());
15760        queue.put(&mut owner).unwrap();
15761        runner.state.driver_exited = true;
15762        runner.state.task_instruction = Some(owner.instruction.clone());
15763        runner.state.save().unwrap();
15764        let _claim = queue.claim(&task).unwrap();
15765        let reserve = |id: &str| {
15766            let directory = home.join("workers/jobs").join(id);
15767            std::fs::create_dir_all(&directory).unwrap();
15768            let job = crate::worker::Job {
15769                id: id.into(),
15770                sequence: 1,
15771                kind: "task".into(),
15772                key: format!("task:{task}"),
15773                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
15774                generation: "fixture-generation".into(),
15775                version: None,
15776                status: crate::worker::Status::Running,
15777                result: None,
15778                pid: Some(std::process::id()),
15779                created_at: 0,
15780                finished_at: None,
15781            };
15782            std::fs::write(
15783                directory.join("job.json"),
15784                serde_json::to_vec(&job).unwrap(),
15785            )
15786            .unwrap();
15787            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
15788        };
15789        let _own = reserve("abcdef");
15790        let other = reserve("fedcba");
15791        assert!(
15792            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15793                .await
15794                .is_err()
15795        );
15796        assert!(
15797            RunState::load(&runner.state.id)
15798                .unwrap()
15799                .base_sync_recovery
15800                .authorizations
15801                .is_empty()
15802        );
15803        drop(other);
15804        assert!(crate::daemon::is_working_on(
15805            &home,
15806            &runner.state.id,
15807            Timestamp::now()
15808        ));
15809        assert!(
15810            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
15811        );
15812        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15813            .await
15814            .unwrap();
15815        assert!(
15816            RunState::load(&runner.state.id)
15817                .unwrap()
15818                .base_recovery_pending()
15819        );
15820    }
15821
15822    #[cfg(unix)]
15823    #[tokio::test]
15824    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
15825        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15826        let home = crate::run::home();
15827        runner.state.driver_pid = Some(std::process::id());
15828        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
15829        runner.state.driver_exited = false;
15830        runner.state.begin_pass();
15831        runner.state.save().unwrap();
15832        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
15833        driver.record(&runner.state).unwrap();
15834        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
15835        // Preserve the durable claim as if the process died before Drop saved.
15836        drop(driver);
15837        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
15838        assert_eq!(
15839            runner.state.liveness_under(&home, false),
15840            crate::run::Liveness::Dead
15841        );
15842        let queue = crate::queue::Queue::open();
15843        let _claim = queue.claim(&task).unwrap();
15844        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15845            .await
15846            .unwrap();
15847        let disk = RunState::load(&runner.state.id).unwrap();
15848        assert!(!disk.driver_exited);
15849        assert!(disk.base_recovery_pending());
15850    }
15851
15852    #[cfg(unix)]
15853    #[tokio::test]
15854    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
15855        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
15856        if std::env::var_os(CHILD).is_none() {
15857            use std::os::unix::fs::PermissionsExt;
15858            let tmp = tempfile::tempdir().unwrap();
15859            let gh = tmp.path().join("gh");
15860            std::fs::write(&gh, r#"#!/bin/sh
15861case "$1 $2" in
15862  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
15863  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
15864  *) exit 2 ;;
15865esac
15866"#).unwrap();
15867            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
15868            let mut paths = vec![tmp.path().to_path_buf()];
15869            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
15870            let output = std::process::Command::new(std::env::current_exe().unwrap())
15871                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
15872                .env(CHILD, "1")
15873                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
15874                .env("PATH", std::env::join_paths(paths).unwrap())
15875                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
15876                .output().unwrap();
15877            assert!(
15878                output.status.success()
15879                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15880                "{}\n{}",
15881                String::from_utf8_lossy(&output.stdout),
15882                String::from_utf8_lossy(&output.stderr)
15883            );
15884            return;
15885        }
15886        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15887        let repo = runner.state.repo.clone();
15888        git::git(&repo, &["checkout", "main"]).await.unwrap();
15889        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
15890        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15891        git::git(&repo, &["checkout", "fix/retained"])
15892            .await
15893            .unwrap();
15894        runner.state.pr = Some(crate::run::PrRecord {
15895            url: "https://example.test/project/pull/1".into(),
15896            number: 1,
15897            state: "open".into(),
15898            checks: "unknown".into(),
15899            round: 0,
15900            rounds: 1,
15901            red_at_merge: Vec::new(),
15902        });
15903        runner.sync_to_base().await.unwrap();
15904        assert_eq!(runner.state.status, RunStatus::Blocked);
15905        assert_eq!(
15906            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
15907            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
15908        );
15909        assert!(runner.state.base_sync_closure_pending());
15910        assert!(crate::handover::can_resume(&runner.state));
15911        let queue = crate::queue::Queue::open();
15912        assert!(!crate::base_sync_recovery::held_task(
15913            &queue,
15914            &queue.get(&task).unwrap()
15915        ));
15916        assert!(runner.state.authorize_base_sync(&task).is_err());
15917        // Legacy closure stops have no typed stop but must retain the checkpoint.
15918        runner.state.base_sync_recovery.stop = None;
15919        runner.state.status = RunStatus::Voting;
15920        runner.state.gate_ran = true;
15921        runner.state.config.graph.review_rounds = 0;
15922        runner.state.save().unwrap();
15923        assert!(crate::handover::can_resume(&runner.state));
15924        runner.state = RunState::load(&runner.state.id).unwrap();
15925        runner.execute_graph().await.unwrap();
15926        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
15927        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
15928        assert_eq!(
15929            runner.state.base_sync.as_ref().unwrap().attempts,
15930            BASE_SYNC_ROUNDS
15931        );
15932        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
15933        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15934        assert!(runner.state.reviews.is_empty());
15935        assert!(runner.state.gate.is_empty());
15936    }
15937
15938    #[cfg(unix)]
15939    #[tokio::test]
15940    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
15941        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15942        runner.state.driver_exited = true;
15943        runner.state.save().unwrap();
15944        let home = crate::run::home();
15945        let queue = crate::queue::Queue::open();
15946        let _claim = queue.claim(&task).unwrap();
15947        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
15948        assert!(
15949            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15950                .await
15951                .is_err()
15952        );
15953        drop(lease);
15954        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
15955        assert!(
15956            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15957                .await
15958                .is_err()
15959        );
15960        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
15961        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15962            .await
15963            .unwrap();
15964        let disk = RunState::load(&runner.state.id).unwrap();
15965        assert!(disk.base_recovery_pending());
15966        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
15967        assert!(
15968            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15969                .await
15970                .is_err()
15971        );
15972        assert_eq!(
15973            RunState::load(&runner.state.id)
15974                .unwrap()
15975                .base_sync_recovery
15976                .authorizations
15977                .len(),
15978            1
15979        );
15980        let mut owner = queue.get(&task).unwrap();
15981        owner.runs.push("20260101-000000-ffff".into());
15982        queue.put(&mut owner).unwrap();
15983        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15984            .await
15985            .unwrap_err();
15986        assert!(error.to_string().contains("run task ownership changed"));
15987    }
15988
15989    #[cfg(unix)]
15990    #[tokio::test]
15991    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
15992        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
15993        let mut legacy = serde_json::to_value(&runner.state).unwrap();
15994        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
15995        legacy["status"] = serde_json::json!("voting");
15996        runner.state = serde_json::from_value(legacy).unwrap();
15997        runner.state.save().unwrap();
15998        for _ in 0..3 {
15999            runner.execute_graph().await.unwrap();
16000            assert_eq!(runner.state.status, RunStatus::Blocked);
16001            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16002            assert!(!crate::handover::can_resume(&runner.state));
16003            assert_eq!(
16004                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16005                head
16006            );
16007            assert!(runner.state.reviews.is_empty());
16008            assert!(!runner.state.gate_ran);
16009        }
16010        let notice = crate::notices::run_ended(&runner.state).unwrap();
16011        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16012        runner.state.begin_pass();
16013        runner.state.status = RunStatus::Voting;
16014        runner.deliberate().await.unwrap();
16015        runner.vote().await.unwrap();
16016        runner.finish_after_tally().await.unwrap();
16017        assert_eq!(runner.state.status, RunStatus::Blocked);
16018        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16019        assert_eq!(
16020            crate::notices::run_ended(&runner.state).unwrap().since,
16021            notice.since
16022        );
16023    }
16024
16025    #[cfg(unix)]
16026    #[tokio::test]
16027    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16028        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16029        for detail in [
16030            "fix/retained conflicts with origin/main and did not rebase: conflict",
16031            "rebased fix/retained locally but origin refused the push",
16032            "unknown failure",
16033        ] {
16034            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16035            runner.state.record_base_sync_stop(&head);
16036            assert!(runner.state.authorize_base_sync(&task).is_err());
16037            runner.state.status = RunStatus::Voting;
16038            runner.execute_graph().await.unwrap();
16039            assert_eq!(runner.state.status, RunStatus::Blocked);
16040            assert!(!crate::handover::can_resume(&runner.state));
16041            assert_eq!(
16042                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16043                head
16044            );
16045        }
16046    }
16047
16048    #[cfg(unix)]
16049    #[tokio::test]
16050    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
16051        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16052        runner.state.authorize_base_sync(&task).unwrap();
16053        assert!(runner.state.consume_base_recovery(&head));
16054        runner.state.save().unwrap();
16055        runner.state = RunState::load(&runner.state.id).unwrap();
16056        runner.execute_graph().await.unwrap();
16057        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16058        assert_eq!(runner.state.status, RunStatus::Blocked);
16059        assert!(!runner.state.consume_base_recovery(&head));
16060        assert!(!crate::handover::can_resume(&runner.state));
16061        runner.state.authorize_base_sync(&task).unwrap();
16062        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
16063        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16064        assert_eq!(
16065            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16066            head
16067        );
16068    }
16069
16070    #[cfg(unix)]
16071    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
16072        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
16073        // This is an ordinary gate stop, not an authorized base-budget recovery.
16074        runner.state.prep_complete = Some(true);
16075        runner.state.base_sync = None;
16076        runner.state.base_sync_recovery = Default::default();
16077        runner.state.config.graph.gate_fix_rounds = 1;
16078        let mut review = review_round(true, 0, 1, 1, false, true);
16079        review.head = head.clone();
16080        review.verified_head = Some(head.clone());
16081        runner.state.reviews = vec![review];
16082        runner.state.gate = vec![CommandOutcome {
16083            command: "old gate".into(),
16084            code: Some(1),
16085            output_tail: "old failure".into(),
16086            duration_ms: 0,
16087            resource_blocked: false,
16088        }];
16089        runner.state.gate_head = Some(head.clone());
16090        runner.state.gate_ran = true;
16091        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
16092            agent: "fixture".into(),
16093            failed: runner.state.gate.clone(),
16094            notes: "spent".into(),
16095            committed: false,
16096            error: None,
16097        }];
16098        runner.state.land_approval = Some(crate::run::LandApproval {
16099            question: "old approval".into(),
16100            head: head.clone(),
16101        });
16102        runner.state.status = RunStatus::Blocked;
16103        (tmp, runner, head)
16104    }
16105
16106    #[cfg(unix)]
16107    #[tokio::test]
16108    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
16109        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
16110            let (tmp, mut runner, old) = resumed_gate_fixture().await;
16111            let phases = tmp.path().join("phases");
16112            runner.state.config.verify.e2e = vec![format!(
16113                "echo e2e >> '{}'; {}",
16114                phases.display(),
16115                if e2e_ok {
16116                    "true"
16117                } else {
16118                    "echo new-e2e-failure; false"
16119                }
16120            )];
16121            runner.state.config.verify.gate = vec![format!(
16122                "echo gate >> '{}'; {}",
16123                phases.display(),
16124                if gate_ok {
16125                    "true"
16126                } else {
16127                    "echo new-gate-failure; false"
16128                }
16129            )];
16130            runner.state.save().unwrap();
16131            runner.state = RunState::load(&runner.state.id).unwrap();
16132            runner.sync_to_base().await.unwrap();
16133            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16134            assert_ne!(head, old);
16135            // Reentry after Git updated but before verification was invalidated.
16136            runner.state = RunState::load(&runner.state.id).unwrap();
16137            runner.execute_graph().await.unwrap();
16138            assert_eq!(runner.state.gate_fixes.len(), 1);
16139            assert_eq!(runner.state.reviews.len(), 1);
16140            assert_eq!(runner.state.reviews[0].head, old);
16141            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
16142            assert_eq!(runner.state.verification_history.len(), 1);
16143            let history = &runner.state.verification_history[0];
16144            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
16145            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
16146            assert!(history.archived_at.is_some());
16147            assert_eq!(history.fixes_spent, Some(1));
16148            assert_eq!(history.gate[0].output_tail, "old failure");
16149            let archive_events: Vec<_> = runner
16150                .state
16151                .events
16152                .iter()
16153                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
16154                .collect();
16155            assert_eq!(archive_events.len(), 1);
16156            assert!(archive_events[0].message.contains(&short(&old)));
16157            assert!(archive_events[0].message.contains(&short(&head)));
16158            assert!(
16159                archive_events[0]
16160                    .message
16161                    .contains("1 gate-fix round(s) already spent stay spent")
16162            );
16163            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
16164            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
16165            if e2e_ok {
16166                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16167                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
16168            } else {
16169                assert!(!runner.state.gate_ran);
16170            }
16171            assert_eq!(
16172                runner.state.status,
16173                if e2e_ok && gate_ok {
16174                    RunStatus::Ready
16175                } else {
16176                    RunStatus::Blocked
16177                }
16178            );
16179            if !e2e_ok || !gate_ok {
16180                let cause = runner.state.verification_blocking_cause().unwrap();
16181                assert!(cause.contains(&head));
16182                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
16183                // A later successful progress event must not become the cause.
16184                runner.state.event("land", "rebased successfully");
16185                let notice = crate::notices::run_ended(&runner.state).unwrap();
16186                assert!(!format!("{notice:?}").contains("rebased successfully"));
16187                assert!(runner.state.merge.is_none());
16188            }
16189            let before = std::fs::read_to_string(&phases).unwrap();
16190            runner.state.save().unwrap();
16191            runner.state = RunState::load(&runner.state.id).unwrap();
16192            runner.review_loop().await.unwrap();
16193            runner.gate().await.unwrap();
16194            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
16195            assert_eq!(runner.state.verification_history.len(), 1);
16196            let history = &runner.state.verification_history[0];
16197            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
16198            assert!(history.archived_at.is_some());
16199            assert_eq!(history.fixes_spent, Some(1));
16200        }
16201    }
16202
16203    #[cfg(unix)]
16204    #[tokio::test]
16205    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
16206        let (tmp, mut runner, head) = resumed_gate_fixture().await;
16207        let marker = tmp.path().join("gate-ran");
16208        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
16209        runner.review_loop().await.unwrap();
16210        runner.gate().await.unwrap();
16211        assert_eq!(runner.state.status, RunStatus::Blocked);
16212        assert!(!marker.exists());
16213        assert!(runner.state.verification_history.is_empty());
16214        runner.state.gate_head = None;
16215        runner.gate().await.unwrap();
16216        assert!(marker.exists());
16217        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16218        assert_eq!(runner.state.verification_history.len(), 1);
16219        assert_eq!(runner.state.gate_fixes.len(), 1);
16220    }
16221
16222    #[cfg(unix)]
16223    #[tokio::test]
16224    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
16225        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16226        runner.sync_to_base().await.unwrap();
16227        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16228        runner.state.reconcile_verification_head(&head);
16229        // Both crash-before-command and resource wait are unresolved evidence.
16230        for outcomes in [
16231            Vec::new(),
16232            vec![CommandOutcome {
16233                command: "true".into(),
16234                code: None,
16235                output_tail: "cache unavailable".into(),
16236                duration_ms: 0,
16237                resource_blocked: true,
16238            }],
16239        ] {
16240            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16241                head: head.clone(),
16242                passed: false,
16243                outcomes,
16244            });
16245            runner.state.gate_ran = false;
16246            runner.state.gate.clear();
16247            runner.state.save().unwrap();
16248            runner.state = RunState::load(&runner.state.id).unwrap();
16249            runner.merge().await.unwrap();
16250            assert!(runner.state.merge.is_none());
16251            runner.review_loop().await.unwrap();
16252            runner.gate().await.unwrap();
16253            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
16254            assert!(runner.state.gate_allows_progress());
16255            assert_eq!(runner.state.gate_fixes.len(), 1);
16256            assert!(runner.state.merge.is_none());
16257        }
16258        // Moving the winner after a successful gate cannot publish an unchecked SHA.
16259        git::git(
16260            &runner.state.repo,
16261            &["commit", "--allow-empty", "-m", "external update"],
16262        )
16263        .await
16264        .unwrap();
16265        runner.merge().await.unwrap();
16266        assert!(runner.state.merge.is_none());
16267        assert_eq!(runner.state.status, RunStatus::Blocked);
16268    }
16269
16270    #[cfg(unix)]
16271    #[tokio::test]
16272    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
16273        for waiting_on_gate in [false, true] {
16274            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
16275            runner.sync_to_base().await.unwrap();
16276            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16277            runner.state.reconcile_verification_head(&head);
16278            if waiting_on_gate {
16279                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16280                    head: head.clone(),
16281                    passed: true,
16282                    outcomes: Vec::new(),
16283                });
16284            }
16285            let cache = tmp.path().join("cache");
16286            runner.state.config.verify.e2e =
16287                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
16288            runner.state.config.verify.gate = vec!["true".into()];
16289            runner.state.config.graph.timeout_resource_wait = 1;
16290            let owner =
16291                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
16292            let guard =
16293                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
16294                    crate::cache::AcquireOutcome::Acquired(g) => g,
16295                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
16296                };
16297            runner.review_loop().await.unwrap();
16298            runner.gate().await.unwrap();
16299            assert!(!runner.state.gate_ran);
16300            assert_eq!(runner.state.gate_fixes.len(), 1);
16301            assert!(runner.state.resource_block.is_some());
16302            runner.merge().await.unwrap();
16303            assert!(runner.state.merge.is_none());
16304            drop(guard);
16305            runner.state = RunState::load(&runner.state.id).unwrap();
16306            // The driver clears the previous pass's resource stop on resume.
16307            runner.state.resource_block = None;
16308            runner.review_loop().await.unwrap();
16309            runner.gate().await.unwrap();
16310            assert!(runner.state.gate_allows_progress());
16311            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16312            assert_eq!(runner.state.gate_fixes.len(), 1);
16313        }
16314    }
16315
16316    #[cfg(unix)]
16317    #[tokio::test]
16318    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
16319        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16320        runner.sync_to_base().await.unwrap();
16321        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16322        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
16323        // Verification passes before pre_gate and fails on the mechanically changed head.
16324        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
16325        runner.review_loop().await.unwrap();
16326        runner.gate().await.unwrap();
16327        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16328        assert_ne!(before, after);
16329        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
16330        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
16331        assert!(!runner.state.gate_allows_progress());
16332        runner.merge().await.unwrap();
16333        assert!(runner.state.merge.is_none());
16334        assert!(
16335            runner
16336                .state
16337                .verification_history
16338                .iter()
16339                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
16340        );
16341
16342        // An incomplete review is still an independent hold on a changed head.
16343        runner.state.reviews[0].clean = false;
16344        runner.state.reviews[0].answered = 0;
16345        runner.state.config.verify.e2e = vec!["true".into()];
16346        runner.review_loop().await.unwrap();
16347        runner.gate().await.unwrap();
16348        runner.merge().await.unwrap();
16349        assert_eq!(runner.state.status, RunStatus::Blocked);
16350        assert!(runner.state.merge.is_none());
16351        assert!(
16352            runner
16353                .state
16354                .verification_blocking_cause()
16355                .unwrap()
16356                .contains("review blocked")
16357        );
16358    }
16359
16360    #[cfg(unix)]
16361    #[tokio::test]
16362    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
16363        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
16364        runner.state.gate_head = None;
16365        runner.state.config.verify.gate =
16366            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
16367        runner.review_loop().await.unwrap();
16368        runner.gate().await.unwrap();
16369        assert_ne!(
16370            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16371            old
16372        );
16373        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
16374        assert!(!runner.state.gate_allows_progress());
16375        assert_eq!(runner.state.status, RunStatus::Blocked);
16376        runner.merge().await.unwrap();
16377        assert!(runner.state.merge.is_none());
16378    }
16379
16380    #[cfg(unix)]
16381    #[tokio::test]
16382    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
16383        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16384        let mut old_review = review_round(true, 0, 1, 1, false, true);
16385        old_review.head = head.clone();
16386        old_review.verified_head = Some(head.clone());
16387        runner.state.reviews = vec![old_review];
16388        runner.state.gate = vec![CommandOutcome {
16389            command: "old gate".into(),
16390            code: None,
16391            output_tail: "timed out after 2400s".into(),
16392            duration_ms: 2_400_000,
16393            resource_blocked: false,
16394        }];
16395        runner.state.gate_ran = true;
16396        runner.state.gate_head = Some(head.clone());
16397        runner.state.land_approval = Some(crate::run::LandApproval {
16398            question: "old-head-approval".into(),
16399            head: head.clone(),
16400        });
16401        runner.state.authorize_base_sync(&task).unwrap();
16402        runner.sync_to_base().await.unwrap();
16403        assert!(!runner.state.base_sync_blocked());
16404        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16405        assert_ne!(head, fresh);
16406        assert!(runner.state.reviews.is_empty());
16407        assert!(!runner.state.gate_ran);
16408        let saved = &runner.state.base_sync_recovery.authorizations[0];
16409        assert_eq!(saved.reviews[0].head, head);
16410        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
16411        assert!(saved.gate_ran);
16412        assert!(!saved.gate[0].ok());
16413        runner.review_loop().await.unwrap();
16414        runner.gate().await.unwrap();
16415        assert_eq!(runner.state.reviews[0].head, fresh);
16416        assert_eq!(
16417            runner.state.reviews[0].verified_head.as_deref(),
16418            Some(fresh.as_str())
16419        );
16420        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
16421        assert!(runner.state.gate_ran);
16422        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16423        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
16424        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
16425        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16426        assert_eq!(
16427            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
16428            "retained change"
16429        );
16430    }
16431
16432    #[cfg(unix)]
16433    #[tokio::test]
16434    async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
16435        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16436        let queue = crate::queue::Queue::open();
16437        let mut owner = queue.get(&task).unwrap();
16438        owner.release_operator("test-resume", None);
16439        queue.put(&mut owner).unwrap();
16440        let generation = owner.resumption.as_ref().unwrap().generation;
16441        runner.state.task_instruction = Some(owner.instruction.clone());
16442        runner.state.driver_exited = true;
16443        runner.state.save().unwrap();
16444        let home = crate::run::home();
16445        let _claim = queue.claim(&task).unwrap();
16446        owner.resumption.as_mut().unwrap().at =
16447            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
16448        queue.put(&mut owner).unwrap();
16449        assert!(
16450            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16451                .await
16452                .is_err(),
16453            "an approval from before the stop grants nothing"
16454        );
16455        owner.resumption.as_mut().unwrap().at = Timestamp::now();
16456        queue.put(&mut owner).unwrap();
16457        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16458            .await
16459            .unwrap();
16460        assert!(
16461            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16462                .await
16463                .is_err()
16464        );
16465        runner.state = RunState::load(&runner.state.id).unwrap();
16466        assert_eq!(
16467            runner.state.base_sync_recovery.authorizations[0].generation,
16468            Some(generation)
16469        );
16470        runner.sync_to_base().await.unwrap();
16471        let repo = runner.state.repo.clone();
16472        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16473        git::git(&repo, &["checkout", "main"]).await.unwrap();
16474        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
16475        git::git(&repo, &["add", "."]).await.unwrap();
16476        git::git(&repo, &["commit", "-m", "move base again"])
16477            .await
16478            .unwrap();
16479        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16480        git::git(&repo, &["checkout", "fix/retained"])
16481            .await
16482            .unwrap();
16483        runner.finish_after_tally().await.unwrap();
16484        assert_eq!(runner.state.status, RunStatus::Blocked);
16485        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16486        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16487        assert!(!runner.state.base_recovery_pending());
16488        assert!(!crate::handover::can_resume(&runner.state));
16489        assert!(!runner.state.gate_ran);
16490        // Even replaying the generation with a later timestamp grants nothing.
16491        owner.resumption.as_mut().unwrap().at = Timestamp::now();
16492        queue.put(&mut owner).unwrap();
16493        assert!(
16494            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16495                .await
16496                .is_err()
16497        );
16498        runner.state = RunState::load(&runner.state.id).unwrap();
16499        runner.sync_to_base().await.unwrap();
16500        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16501        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
16502        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16503    }
16504
16505    #[cfg(unix)]
16506    #[tokio::test]
16507    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
16508        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
16509        runner.state.base_sync = None;
16510        runner.state.status = RunStatus::Blocked;
16511        let mut second = runner.state.candidates[0].clone();
16512        second.label = 'B';
16513        runner.state.candidates.push(second);
16514        assert!(runner.state.votes.is_empty());
16515        assert!(runner.state.deliberation.is_empty());
16516        runner.deliberate().await.unwrap();
16517        runner.vote().await.unwrap();
16518        assert_eq!(runner.state.status, RunStatus::Blocked);
16519        assert!(runner.state.votes.is_empty());
16520        assert!(runner.state.deliberation.is_empty());
16521    }
16522
16523    /// A minimal, git-free `Runner` at a given status — `settle_questions`
16524    /// reads nothing else off it.
16525    fn runner_at(status: RunStatus) -> Runner {
16526        let mut state = RunState::new(
16527            PathBuf::from("/nonexistent/repo"),
16528            "main".to_owned(),
16529            "deadbeef".to_owned(),
16530            "task".to_owned(),
16531            Config::default(),
16532        );
16533        // These fixtures share one home and save concurrently. Short clock
16534        // IDs can collide, making an ownership test read another test's run.
16535        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
16536        state.id = format!(
16537            "20000101-000000-{:04x}",
16538            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
16539        );
16540        state.status = status;
16541        Runner {
16542            state,
16543            roles: ResolvedRoles {
16544                implementers: Vec::new(),
16545                judges: Vec::new(),
16546                reviewers: Vec::new(),
16547                fixer: None,
16548                conductor: conductor(),
16549                implementer_roster: Vec::new(),
16550                judge_roster: Vec::new(),
16551                reviewer_roster: Vec::new(),
16552            },
16553            sem: Arc::new(Semaphore::new(1)),
16554            pause: Pause::new(),
16555            interrupt: Pause::new(),
16556        }
16557    }
16558
16559    #[tokio::test]
16560    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
16561        crate::run::pin_test_home();
16562        let mut runner = runner_at(RunStatus::Gating);
16563        runner.state.save().unwrap();
16564        let id = runner.state.id.clone();
16565        let passes = runner.state.passes.len();
16566        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16567        let mut claiming = Box::pin(runner.claim_driver());
16568        std::future::poll_fn(|cx| {
16569            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
16570            std::task::Poll::Ready(())
16571        })
16572        .await;
16573        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
16574        drop(owner);
16575        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
16576            .await
16577            .unwrap()
16578            .unwrap();
16579        assert_eq!(execution.runner.state.passes.len(), passes + 1);
16580        assert!(execution.runner.state.resource_block.is_none());
16581        assert_eq!(
16582            execution.runner.state.liveness(false),
16583            crate::run::Liveness::Live
16584        );
16585        drop(execution);
16586        assert!(RunState::load(&id).unwrap().driver_exited);
16587    }
16588
16589    #[tokio::test]
16590    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
16591        crate::run::pin_test_home();
16592        let mut runner = runner_at(RunStatus::Gating);
16593        runner.state.save().unwrap();
16594        // run.json can commit, but the subsequent lease metadata write fails.
16595        // Cleanup must run after releasing the short ownership lock.
16596        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
16597        assert!(runner.claim_driver().await.is_err());
16598        assert!(runner.state.driver_exited);
16599        assert_eq!(runner.state.driver_pid, None);
16600        let disk = RunState::load(&runner.state.id).unwrap();
16601        assert!(disk.driver_exited);
16602        assert_eq!(disk.driver_pid, None);
16603        assert_eq!(disk.status, RunStatus::Gating);
16604    }
16605
16606    #[tokio::test]
16607    async fn an_early_graph_error_releases_its_driver() {
16608        crate::run::pin_test_home();
16609        let mut runner = runner_at(RunStatus::Blocked);
16610        runner.state.config.graph.selected_implementer = Some("selected".into());
16611        runner.state.save().unwrap();
16612        assert!(runner.execute().await.is_err());
16613        let disk = RunState::load(&runner.state.id).unwrap();
16614        assert!(disk.driver_exited);
16615        assert_eq!(disk.driver_pid, None);
16616        assert_eq!(disk.status, RunStatus::Blocked);
16617    }
16618
16619    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
16620        crate::run::MergeRepair {
16621            question: "repair-question".into(),
16622            head: "old-head".into(),
16623            pr: "https://github.com/o/r/pull/42".into(),
16624            reviews: vec![review_round(false, 1, 1, 1, false, true)],
16625            gate: vec![outcome(Some(0), false)],
16626            gate_head: Some("old-head".into()),
16627            gate_fixes: Vec::new(),
16628            pre_gate: Vec::new(),
16629            phase,
16630            failure: None,
16631        }
16632    }
16633
16634    #[tokio::test]
16635    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
16636        crate::run::pin_test_home();
16637        let mut runner = runner_at(RunStatus::Landing);
16638        runner.state.merge_repairs.push(merge_repair_request(
16639            crate::run::MergeRepairPhase::Requested,
16640        ));
16641        runner.state.save().unwrap();
16642        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
16643        assert!(
16644            runner
16645                .execute()
16646                .await
16647                .unwrap_err()
16648                .to_string()
16649                .contains("already running")
16650        );
16651        assert_eq!(
16652            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16653            crate::run::MergeRepairPhase::Requested
16654        );
16655        drop(claim);
16656        let mut disk = runner.state.clone();
16657        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
16658        disk.save().unwrap();
16659        assert!(
16660            runner
16661                .execute()
16662                .await
16663                .unwrap_err()
16664                .to_string()
16665                .contains("decision changed")
16666        );
16667        assert_eq!(
16668            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16669            crate::run::MergeRepairPhase::Complete
16670        );
16671    }
16672
16673    #[test]
16674    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
16675        crate::run::pin_test_home();
16676        let mut runner = runner_at(RunStatus::Landing);
16677        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
16678        runner.state.reviews = request.reviews.clone();
16679        runner.state.gate = request.gate.clone();
16680        runner.state.gate_ran = true;
16681        runner.state.land_approval = Some(crate::run::LandApproval {
16682            question: "old".into(),
16683            head: "old-head".into(),
16684        });
16685        runner.state.merge_repairs.push(request);
16686        runner.begin_merge_repair_review(0).unwrap();
16687        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
16688        assert!(!runner.state.gate_status().ok());
16689        assert!(runner.state.land_approval.is_none());
16690        let disk = RunState::load(&runner.state.id).unwrap();
16691        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
16692        assert!(disk.merge_repairs[0].gate[0].ok());
16693        assert_eq!(
16694            disk.merge_repairs[0].phase,
16695            crate::run::MergeRepairPhase::Reviewing
16696        );
16697        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
16698        assert!(disk.reviews.is_empty());
16699    }
16700
16701    #[tokio::test]
16702    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
16703        crate::run::pin_test_home();
16704        for phase in [
16705            crate::run::MergeRepairPhase::Fixing,
16706            crate::run::MergeRepairPhase::Requested,
16707        ] {
16708            let mut runner = runner_at(RunStatus::Landing);
16709            runner.state.config.graph.land_rounds = 0;
16710            runner.state.merge_repairs.push(merge_repair_request(phase));
16711            runner.state.save().unwrap();
16712            assert!(!runner.repair_merge_review().await.unwrap());
16713            assert_eq!(runner.state.status, RunStatus::Blocked);
16714            assert_eq!(
16715                runner.state.merge_repairs[0].phase,
16716                crate::run::MergeRepairPhase::Failed
16717            );
16718            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
16719            assert!(
16720                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
16721                    "interrupted"
16722                } else {
16723                    "budget exhausted"
16724                })
16725            );
16726            assert!(runner.state.merge.is_none());
16727        }
16728    }
16729
16730    /// `park_here` folding in the reason `Pause::park_because` recorded -
16731    /// this is what lets an operator reading a run's events tell an
16732    /// interrupt-driven park from an ordinary shutdown park.
16733    #[test]
16734    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
16735        crate::run::pin_test_home();
16736        let mut runner = runner_at(RunStatus::Implementing);
16737        let interrupt = Pause::new();
16738        runner.watch_interrupt(interrupt.clone());
16739
16740        interrupt.park_because("task a1b2 asked to run first");
16741
16742        assert!(runner.park_here().expect("park_here"));
16743        assert!(runner.state.parked);
16744        let last = runner.state.events.last().expect("a park event");
16745        assert_eq!(last.node, "park");
16746        assert!(
16747            last.message.contains("task a1b2 asked to run first"),
16748            "expected the interrupt reason in {:?}",
16749            last.message
16750        );
16751    }
16752
16753    /// The cause is stored on the pass when it parks, and the first cause a
16754    /// shared pause was given is the one that stays.
16755    #[test]
16756    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
16757        crate::run::pin_test_home();
16758        let mut runner = runner_at(RunStatus::Implementing);
16759        runner.state.begin_pass();
16760        let shutdown = Pause::new();
16761        runner.on_pause(shutdown.clone());
16762        let upgrade = ParkCause::Upgrade {
16763            from: "0.121.0".to_owned(),
16764            to: Some("0.121.1".to_owned()),
16765        };
16766        shutdown.park_for(upgrade.clone());
16767        shutdown.park_for(ParkCause::Operator);
16768
16769        assert!(runner.park_here().expect("park_here"));
16770        assert_eq!(
16771            runner.state.passes[0].end,
16772            Some(crate::run::PassEnd::Parked { cause: upgrade })
16773        );
16774        let last = runner.state.events.last().expect("a park event");
16775        assert!(last.message.contains("0.121.0"), "{}", last.message);
16776    }
16777
16778    #[tokio::test]
16779    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
16780        crate::run::pin_test_home();
16781        let mut runner = runner_at(RunStatus::Reviewing);
16782        runner.state.save().unwrap();
16783        let mut execution = runner.claim_driver().await.expect("first execution");
16784        assert_eq!(execution.runner.state.passes.len(), 1);
16785        execution
16786            .driver
16787            .finish(&mut execution.runner.state)
16788            .await
16789            .unwrap();
16790    }
16791
16792    #[tokio::test]
16793    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
16794        crate::run::pin_test_home();
16795        let mut runner = runner_at(RunStatus::Reviewing);
16796        runner.state.config.agents = vec![conductor()];
16797        runner.state.config.graph.timeout_resource_wait = 0;
16798        runner.state.save().unwrap();
16799        let id = runner.state.id.clone();
16800        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16801        assert!(runner.claim_driver().await.is_err());
16802        drop(owner);
16803        drop(runner);
16804        let mut resumed = Runner::resume(&id).unwrap();
16805        assert_eq!(resumed.state.driver_claimed, Some(false));
16806        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
16807        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
16808        let mut execution = resumed
16809            .claim_driver()
16810            .await
16811            .expect("retry initial claim after reload");
16812        assert_eq!(execution.runner.state.driver_claimed, Some(true));
16813        assert_eq!(execution.runner.state.passes.len(), 1);
16814        execution
16815            .driver
16816            .finish(&mut execution.runner.state)
16817            .await
16818            .unwrap();
16819    }
16820
16821    #[tokio::test]
16822    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
16823        crate::run::pin_test_home();
16824        let mut runner = runner_at(RunStatus::Reviewing);
16825        runner.state.config.agents = vec![conductor()];
16826        runner.state.save().unwrap();
16827        let id = runner.state.id.clone();
16828        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
16829        let mut resumed = Runner::resume(&id).unwrap();
16830        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
16831        assert!(resumed.claim_driver().await.is_err());
16832        assert!(RunState::load(&id).unwrap().passes.is_empty());
16833        drop(lease);
16834        let mut execution = resumed.claim_driver().await.unwrap();
16835        execution
16836            .driver
16837            .finish(&mut execution.runner.state)
16838            .await
16839            .unwrap();
16840    }
16841
16842    #[tokio::test]
16843    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
16844        crate::run::pin_test_home();
16845        let mut runner = runner_at(RunStatus::Reviewing);
16846        let state = &mut runner.state;
16847        state.config.agents = vec![conductor()];
16848        state.driver_claimed = None;
16849        state.driver_pid = Some(std::process::id());
16850        state.driver_started_at = None;
16851        state.begin_pass();
16852        state.save().unwrap();
16853        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
16854        std::fs::write(
16855            state.dir().join("driver.json"),
16856            serde_json::to_vec(&serde_json::json!({
16857                "pid": state.driver_pid, "started_at": null,
16858                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
16859            }))
16860            .unwrap(),
16861        )
16862        .unwrap();
16863        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
16864        drop(lease);
16865        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
16866        let mut resumed = Runner::resume(&state.id).unwrap();
16867        assert!(!resumed.state.driver_exited);
16868        let mut execution = resumed
16869            .claim_driver()
16870            .await
16871            .expect("lease proves previous execution ended");
16872        assert_eq!(execution.runner.state.passes.len(), 2);
16873        execution
16874            .driver
16875            .finish(&mut execution.runner.state)
16876            .await
16877            .unwrap();
16878    }
16879
16880    #[tokio::test]
16881    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
16882        crate::run::pin_test_home();
16883        let mut runner = runner_at(RunStatus::Reviewing);
16884        let mut json = serde_json::to_value(&runner.state).unwrap();
16885        json.as_object_mut().unwrap().remove("driver_claimed");
16886        runner.state = serde_json::from_value(json).unwrap();
16887        assert_eq!(runner.state.driver_claimed, None);
16888        runner.state.save().unwrap();
16889        let mut execution = runner
16890            .claim_driver()
16891            .await
16892            .expect("the execution lease, not a missing marker, excludes a second driver");
16893        assert_eq!(execution.runner.state.passes.len(), 1);
16894        execution
16895            .driver
16896            .finish(&mut execution.runner.state)
16897            .await
16898            .unwrap();
16899    }
16900
16901    #[tokio::test]
16902    async fn driver_admission_refuses_a_proven_live_driver() {
16903        crate::run::pin_test_home();
16904        let mut runner = runner_at(RunStatus::Reviewing);
16905        let pid = std::process::id();
16906        runner.state.driver_pid = Some(pid);
16907        runner.state.driver_started_at = crate::proc::process_started_at(pid);
16908        assert!(runner.state.driver_started_at.is_some());
16909        runner.state.save().unwrap();
16910        let error = runner
16911            .claim_driver()
16912            .await
16913            .err()
16914            .expect("a live driver must refuse");
16915        assert!(error.to_string().contains("live driver"), "{error:#}");
16916        let saved = RunState::load(&runner.state.id).unwrap();
16917        assert_eq!(saved.driver_pid, Some(pid));
16918        assert_eq!(saved.status, RunStatus::Reviewing);
16919        assert!(saved.passes.is_empty());
16920    }
16921
16922    #[test]
16923    fn parking_resource_halt_persists_the_pass_once() {
16924        crate::run::pin_test_home();
16925        let mut runner = runner_at(RunStatus::Reviewing);
16926        runner.state.begin_pass();
16927        runner.state.resource_block = Some(crate::run::ResourceBlock {
16928            node: "fix".into(),
16929            kind: "shared build cache".into(),
16930            detail: "cancelled".into(),
16931            at: Timestamp::now(),
16932            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
16933        });
16934        let cause = ParkCause::Upgrade {
16935            from: "old".into(),
16936            to: Some("new".into()),
16937        };
16938        runner.pause.park_for(cause.clone());
16939        // Both direct fixer exits and graph resource boundaries must complete
16940        // parking, rather than returning a stale Reviewing record.
16941        runner.stop_for_resource().unwrap();
16942        assert!(runner.resource_halt().unwrap());
16943        assert!(runner.park_here().unwrap());
16944        let saved = RunState::load(&runner.state.id).unwrap();
16945        assert!(saved.parked);
16946        assert_eq!(saved.status, RunStatus::Reviewing);
16947        assert!(saved.passes[0].ended_at.is_some());
16948        assert_eq!(
16949            saved.passes[0].end,
16950            Some(crate::run::PassEnd::Parked {
16951                cause: cause.clone()
16952            })
16953        );
16954        assert_eq!(
16955            saved.resource_block.unwrap().cause,
16956            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
16957        );
16958        assert_eq!(
16959            saved
16960                .events
16961                .iter()
16962                .filter(|event| event.node == "park")
16963                .count(),
16964            1
16965        );
16966    }
16967
16968    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
16969    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
16970    /// cleared) must not make a *different* run - one only watching its own,
16971    /// unshared interrupt `Pause` - see itself as parked. If a future change
16972    /// ever collapsed these back into one handle, the interrupt scheduler
16973    /// would park every run for the rest of the daemon's life, not just the
16974    /// one it meant to interrupt.
16975    #[test]
16976    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
16977        crate::run::pin_test_home();
16978        let mut runner = runner_at(RunStatus::Implementing);
16979        let shutdown = Pause::new();
16980        runner.on_pause(shutdown.clone());
16981        let interrupt = Pause::new();
16982        runner.watch_interrupt(interrupt.clone());
16983
16984        // Nobody has asked for anything yet.
16985        assert!(!runner.park_here().expect("park_here"));
16986        assert!(!runner.state.parked);
16987
16988        // Only the interrupt handle fires; the shutdown handle stays clear.
16989        interrupt.park_because("test");
16990        assert!(!shutdown.parked());
16991        assert!(runner.park_here().expect("park_here"));
16992    }
16993
16994    /// The property every prior attempt at this feature failed to pin down:
16995    /// asking a run to park while one of its nodes has a real, in-flight
16996    /// async operation running (an agent call, in production) must not cut
16997    /// that operation short. `park_here` is only ever consulted *between*
16998    /// `execute`'s node calls - see its own doc - so nothing inside a node
16999    /// can observe a park request until the node itself returns. This proves
17000    /// that structurally, with real `tokio` concurrency and a channel
17001    /// handshake (never a sleep, which would only prove "usually", not
17002    /// "cannot"): the "node" below reports that it has genuinely started,
17003    /// and only then is the park requested; the node still has to be told to
17004    /// finish before `park_here` is ever called, exactly mirroring every
17005    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
17006    /// in `execute`.
17007    #[tokio::test]
17008    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
17009        crate::run::pin_test_home();
17010        let mut runner = runner_at(RunStatus::Implementing);
17011        let interrupt = Pause::new();
17012        runner.watch_interrupt(interrupt.clone());
17013
17014        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
17015        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
17016
17017        // Stands in for one node's in-flight agent call: it proves it has
17018        // genuinely started, then blocks - exactly as a spawned CLI process
17019        // does - until told to finish.
17020        let node = async move {
17021            started_tx.send(()).expect("send started");
17022            finish_rx.await.expect("recv finish");
17023            "node finished"
17024        };
17025
17026        let interrupter = async move {
17027            started_rx.await.expect("recv started");
17028            // The call is now genuinely in flight. Ask it to park.
17029            interrupt.park_because("higher-priority task waiting");
17030            // Nothing the node does can observe this yet - there is no
17031            // check inside it, by construction - so let the executor run
17032            // anything pending and then let the node finish on its own.
17033            tokio::task::yield_now().await;
17034            finish_tx.send(()).expect("send finish");
17035        };
17036
17037        let (node_result, ()) = tokio::join!(node, interrupter);
17038        assert_eq!(
17039            node_result, "node finished",
17040            "the in-flight call ran to completion"
17041        );
17042
17043        // Only now, at the boundary the real `execute` would check right
17044        // after this node, does the park take effect.
17045        assert!(runner.park_here().expect("park_here"));
17046        assert!(runner.state.parked);
17047    }
17048
17049    /// A run parked mid-competition carries every field it had accumulated
17050    /// through the exact same disk round-trip an ordinary resume uses -
17051    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
17052    /// Nothing about parking for an interrupt is a special case of that path;
17053    /// this is what proves it rather than assuming it.
17054    #[test]
17055    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
17056        crate::run::pin_test_home();
17057        let mut runner = runner_at(RunStatus::Judging);
17058        // `Runner::resume` re-resolves roles from the saved config, which
17059        // refuses an empty roster - give it the same minimal one `conductor`
17060        // itself uses.
17061        runner.state.config.agents = vec![conductor()];
17062        runner.state.candidates = vec![Candidate {
17063            index: 0,
17064            label: 'A',
17065            agent: "alpha".to_owned(),
17066            branch: "magi/x/A".to_owned(),
17067            worktree: PathBuf::from("/nonexistent/worktree"),
17068            summary: "did the thing".to_owned(),
17069            stat: "1 file changed".to_owned(),
17070            files: 1,
17071            commits: 1,
17072            empty: false,
17073            failed: None,
17074            verified_noop: None,
17075            duration_ms: 1234,
17076            folded: false,
17077        }];
17078        let run_id = runner.state.id.clone();
17079
17080        let interrupt = Pause::new();
17081        runner.watch_interrupt(interrupt.clone());
17082        interrupt.park_because("task c3d4 asked to run first");
17083        assert!(runner.park_here().expect("park_here"));
17084
17085        let resumed = Runner::resume(&run_id).expect("resume");
17086        assert_eq!(resumed.state.candidates.len(), 1);
17087        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
17088        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
17089        assert_eq!(resumed.state.status, runner.state.status);
17090        assert!(
17091            resumed.state.parked,
17092            "still parked until `execute` actually walks the graph again"
17093        );
17094        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
17095    }
17096
17097    /// A fresh open question on `run`, stored and handed back for assertions.
17098    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
17099        let mut q = ask::Question::new(
17100            run.to_owned(),
17101            "implement".to_owned(),
17102            "impl-A".to_owned(),
17103            "Which storage backend should the cache use?".to_owned(),
17104            String::new(),
17105            vec!["SQLite".to_owned(), "Redis".to_owned()],
17106        );
17107        store.put(&mut q).unwrap();
17108        q
17109    }
17110
17111    #[test]
17112    fn a_failed_runs_open_question_is_abandoned() {
17113        ask_test_home();
17114        let store = ask::Questions::open();
17115        let mut runner = runner_at(RunStatus::Failed);
17116        let run = runner.state.id.clone();
17117        let q = ask_open_question(&store, &run);
17118
17119        runner.settle_questions();
17120
17121        let back = store.get(&q.id).unwrap();
17122        assert!(
17123            !back.status.open(),
17124            "the seat that asked died with the run; nobody is left to read an answer"
17125        );
17126        assert!(
17127            back.detail.contains(&run) && back.detail.contains("failed"),
17128            "the reason names what the run became, not just that it is gone: {}",
17129            back.detail
17130        );
17131    }
17132
17133    #[test]
17134    fn a_merged_runs_open_question_is_abandoned_too() {
17135        ask_test_home();
17136        let store = ask::Questions::open();
17137        // A run that finishes cleanly still leaves nobody to read an answer -
17138        // this is not only a failure-path cleanup.
17139        for status in [RunStatus::Merged, RunStatus::Ready] {
17140            let mut runner = runner_at(status);
17141            let run = runner.state.id.clone();
17142            let q = ask_open_question(&store, &run);
17143
17144            runner.settle_questions();
17145
17146            let back = store.get(&q.id).unwrap();
17147            assert!(
17148                !back.status.open(),
17149                "{status:?} run's question must not outlive the run"
17150            );
17151        }
17152    }
17153
17154    #[test]
17155    fn a_still_resumable_runs_open_question_is_left_alone() {
17156        ask_test_home();
17157        let store = ask::Questions::open();
17158        // `Blocked` and `Stalled` can still be resumed — the candidates, the
17159        // review round and the seat sessions are all still on disk — so a
17160        // question asked mid-round may yet get a real answer from a real
17161        // resume. Sweeping it here would be exactly the failure mode this
17162        // whole feature exists to avoid on the other side.
17163        for status in [RunStatus::Blocked, RunStatus::Stalled] {
17164            let mut runner = runner_at(status);
17165            let run = runner.state.id.clone();
17166            let q = ask_open_question(&store, &run);
17167
17168            runner.settle_questions();
17169
17170            let back = store.get(&q.id).unwrap();
17171            assert!(
17172                back.status.open(),
17173                "{status:?} is still alive; the question must still be waiting"
17174            );
17175        }
17176    }
17177
17178    #[test]
17179    fn settle_questions_never_touches_an_already_answered_question() {
17180        ask_test_home();
17181        let store = ask::Questions::open();
17182        let mut runner = runner_at(RunStatus::Failed);
17183        let run = runner.state.id.clone();
17184        let mut q = ask_open_question(&store, &run);
17185        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
17186            .unwrap();
17187        store.put(&mut q).unwrap();
17188
17189        // Called twice, the way a crash-recovered daemon reclaim and the
17190        // graph's own cleanup both can for the same run — `abandon_for_run`
17191        // only ever touches what is still open, so this must be inert both
17192        // times, not merely the second.
17193        runner.settle_questions();
17194        runner.settle_questions();
17195
17196        let back = store.get(&q.id).unwrap();
17197        assert_eq!(
17198            back.status,
17199            ask::QuestionStatus::Answered,
17200            "a real answer is a decision on record, never overwritten by a sweep"
17201        );
17202    }
17203
17204    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
17205    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
17206    /// without merging, whose winner is still the operator's answer to read.
17207    /// Nothing previously called `fold_run` itself with a real `tally`, so
17208    /// this is the first test to pin down the one distinction the whole
17209    /// automatic-fold feature depends on: the winner's worktree and branch
17210    /// must survive, everything else sharing the run's worktree bay - a
17211    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
17212    /// second sweep treats every non-winner directory under the bay alike -
17213    /// must not.
17214    #[tokio::test]
17215    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
17216        crate::run::pin_test_home();
17217        let tmp = tempfile::tempdir().expect("tempdir");
17218        let repo = tmp.path().join("repo");
17219        std::fs::create_dir_all(&repo).unwrap();
17220        init_repo(&repo);
17221
17222        let mut config = Config::default();
17223        config.graph.worktree_root = Some(tmp.path().join("wt"));
17224
17225        let mut state = RunState::new(
17226            repo.clone(),
17227            "main".to_owned(),
17228            "deadbeef".to_owned(),
17229            "task".to_owned(),
17230            config,
17231        );
17232        let root = state.worktree_root();
17233        let wt_a = root.join("cand-A");
17234        let wt_b = root.join("cand-B");
17235        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
17236            .await
17237            .expect("worktree A");
17238        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
17239            .await
17240            .expect("worktree B");
17241
17242        state.candidates = vec![
17243            Candidate {
17244                index: 0,
17245                label: 'A',
17246                agent: "alpha".to_owned(),
17247                branch: "magi/x/A".to_owned(),
17248                worktree: wt_a.clone(),
17249                summary: String::new(),
17250                stat: String::new(),
17251                files: 0,
17252                commits: 0,
17253                empty: false,
17254                failed: None,
17255                verified_noop: None,
17256                duration_ms: 0,
17257                folded: false,
17258            },
17259            Candidate {
17260                index: 1,
17261                label: 'B',
17262                agent: "beta".to_owned(),
17263                branch: "magi/x/B".to_owned(),
17264                worktree: wt_b.clone(),
17265                summary: String::new(),
17266                stat: String::new(),
17267                files: 0,
17268                commits: 0,
17269                empty: false,
17270                failed: None,
17271                verified_noop: None,
17272                duration_ms: 0,
17273                folded: false,
17274            },
17275        ];
17276        state.tally = Some(Tally {
17277            first_choice: BTreeMap::from([('A', 1)]),
17278            borda: BTreeMap::new(),
17279            winner: 'A',
17280            rankings: 1,
17281            unanimous_initial: true,
17282            deliberated: false,
17283            changed_votes: 0,
17284            unanimous_final: true,
17285            tie_break: None,
17286            judges: 1,
17287            present: 1,
17288            quorum: 1,
17289            met_quorum: true,
17290            uncontested: None,
17291        });
17292        state.status = RunStatus::Ready;
17293
17294        fold_run(&mut state, false, &crate::run::home())
17295            .await
17296            .expect("fold_run");
17297
17298        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
17299        assert!(
17300            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17301            "the unmerged winner's branch survives"
17302        );
17303        assert!(
17304            !state.candidates[0].folded,
17305            "the winner is not marked folded"
17306        );
17307
17308        assert!(!wt_b.exists(), "the loser's worktree is removed");
17309        assert!(
17310            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
17311            "the loser's branch is removed"
17312        );
17313        assert!(state.candidates[1].folded, "the loser is marked folded");
17314    }
17315
17316    #[tokio::test]
17317    #[cfg(unix)]
17318    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
17319        use std::os::fd::AsRawFd as _;
17320        use std::time::Duration;
17321
17322        let tmp = tempfile::tempdir().unwrap();
17323        let repo = tmp.path().join("repo");
17324        std::fs::create_dir_all(&repo).unwrap();
17325        init_repo(&repo);
17326        let common = repo.join(".git");
17327        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
17328        let home = tmp.path().join("home");
17329        let mut config = Config::default();
17330        config.coordination = crate::config::test_coordination();
17331        config.graph.worktree_root = Some(tmp.path().join("wt"));
17332        config.graph.timeout_resource_wait = 1;
17333        let mut state = RunState::new(
17334            repo.clone(),
17335            "main".into(),
17336            "deadbeef".into(),
17337            "task".into(),
17338            config,
17339        );
17340        let root = state.worktree_root();
17341        let candidate = root.join("cand-A");
17342        let reviewer = root.join("review-1");
17343        let branch = state.branch_for('A');
17344        git::worktree_add_branch(&repo, &candidate, &branch, "main")
17345            .await
17346            .unwrap();
17347        git::worktree_add_detached(&repo, &reviewer, "main")
17348            .await
17349            .unwrap();
17350        git::acquire_worktree_config(&repo).await.unwrap();
17351        state.enabled_worktree_config = true;
17352        state.status = RunStatus::Ready;
17353        state.candidates.push(Candidate {
17354            index: 0,
17355            label: 'A',
17356            agent: "implement".into(),
17357            branch: branch.clone(),
17358            worktree: candidate.clone(),
17359            summary: String::new(),
17360            stat: String::new(),
17361            files: 0,
17362            commits: 0,
17363            empty: false,
17364            failed: None,
17365            verified_noop: None,
17366            duration_ms: 0,
17367            folded: false,
17368        });
17369        state.save_under(&home).unwrap();
17370        let config_before = std::fs::read(common.join("config")).unwrap();
17371        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
17372            .await
17373            .unwrap();
17374        // A separate OS handle bypasses the launcher's local gate, exactly as
17375        // an external updater does, and must still prevent all folding work.
17376        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
17377        assert_eq!(
17378            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
17379            0
17380        );
17381        let mut fold = Box::pin(fold_run(&mut state, true, &home));
17382        assert!(
17383            tokio::time::timeout(Duration::from_millis(200), &mut fold)
17384                .await
17385                .is_err()
17386        );
17387        assert!(candidate.exists());
17388        assert!(reviewer.exists());
17389        assert!(git::branch_exists(&repo, &branch).await.unwrap());
17390        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17391        assert_eq!(
17392            git::git(&repo, &["worktree", "list", "--porcelain"])
17393                .await
17394                .unwrap(),
17395            worktrees_before
17396        );
17397        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
17398            .await
17399            .unwrap()
17400            .unwrap_err();
17401        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
17402        drop(fold);
17403        assert!(candidate.exists() && reviewer.exists());
17404        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
17405        assert_eq!(
17406            state.resource_block.as_ref().unwrap().kind,
17407            "repository coordination"
17408        );
17409        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17410        drop(external);
17411        let mut fold = Box::pin(fold_run(&mut state, true, &home));
17412        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
17413            .await
17414            .unwrap()
17415            .unwrap();
17416        drop(fold);
17417        assert!(removed.contains(&branch));
17418        assert!(!root.exists());
17419        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
17420        assert!(state.candidates[0].folded);
17421        assert!(!state.enabled_worktree_config);
17422        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
17423        let enabled = git::git_raw(
17424            &repo,
17425            &["config", "--local", "--get", "extensions.worktreeConfig"],
17426        )
17427        .await
17428        .unwrap();
17429        assert!(!enabled.ok(), "the last config reference was released");
17430    }
17431
17432    /// A branch handed to a later run is that run's (and its pull request's):
17433    /// folding the run that released it must not delete it.
17434    #[tokio::test]
17435    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
17436        let tmp = tempfile::tempdir().expect("tempdir");
17437        let repo = tmp.path().join("repo");
17438        std::fs::create_dir_all(&repo).unwrap();
17439        init_repo(&repo);
17440        let home = tmp.path().join("home");
17441
17442        let mut config = Config::default();
17443        config.graph.worktree_root = Some(tmp.path().join("wt"));
17444        let mut state = RunState::new(
17445            repo.clone(),
17446            "main".to_owned(),
17447            "deadbeef".to_owned(),
17448            "task".to_owned(),
17449            config,
17450        );
17451        // The worktree is already gone (released); the branch survives.
17452        git::git(&repo, &["branch", "magi/x/A", "main"])
17453            .await
17454            .expect("branch");
17455        state.candidates = vec![Candidate {
17456            index: 0,
17457            label: 'A',
17458            agent: "alpha".to_owned(),
17459            branch: "magi/x/A".to_owned(),
17460            worktree: state.worktree_root().join("cand-A"),
17461            summary: String::new(),
17462            stat: String::new(),
17463            files: 0,
17464            commits: 0,
17465            empty: false,
17466            failed: None,
17467            verified_noop: None,
17468            duration_ms: 0,
17469            folded: true,
17470        }];
17471        state.released_to = Some("20260901-000000-new1".to_owned());
17472        state.released_branches = vec!["magi/x/A".to_owned()];
17473
17474        fold_run(&mut state, true, &home).await.expect("fold_run");
17475        state.status = RunStatus::Merged;
17476        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
17477        state.merge = Some(crate::run::MergeOutcome {
17478            mode: MergeMode::Pr,
17479            ok: true,
17480            detail: "confirmed merged".into(),
17481            empty: false,
17482        });
17483        fold_run(&mut state, true, &home)
17484            .await
17485            .expect("merged fold");
17486
17487        assert!(
17488            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17489            "the handed-over branch survives a fold"
17490        );
17491    }
17492
17493    /// A winner with nothing ahead of the base is caught before `gh` is ever
17494    /// asked for a pull request, and the message carries what the task's
17495    /// references resolved to.
17496    #[tokio::test]
17497    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
17498        let tmp = tempfile::tempdir().expect("tempdir");
17499        let repo = tmp.path().join("repo");
17500        std::fs::create_dir_all(&repo).unwrap();
17501        init_repo(&repo);
17502        let run = |args: &[&str]| {
17503            let out = std::process::Command::new("git")
17504                .quiet()
17505                .args(args)
17506                .current_dir(&repo)
17507                .output()
17508                .expect("spawn git");
17509            assert!(out.status.success(), "git {args:?}");
17510        };
17511        run(&["branch", "magi/x/A"]);
17512        run(&["checkout", "-q", "-b", "magi/x/B"]);
17513        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
17514        run(&["add", "-A"]);
17515        run(&["commit", "-q", "-m", "work"]);
17516        run(&["checkout", "-q", "main"]);
17517
17518        // A pull request is compared against the remote's base, so the
17519        // fixture needs one. Before it exists nothing can be read, and the
17520        // answer must be "not empty".
17521        let probe = RunState::new(
17522            repo.clone(),
17523            "main".to_owned(),
17524            "deadbeef".to_owned(),
17525            "task".to_owned(),
17526            Config::default(),
17527        );
17528        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
17529        let bare = tmp.path().join("origin.git");
17530        let out = std::process::Command::new("git")
17531            .quiet()
17532            .args(["init", "-q", "--bare"])
17533            .arg(&bare)
17534            .output()
17535            .expect("spawn git");
17536        assert!(out.status.success(), "git init --bare");
17537        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
17538        run(&["push", "-q", "origin", "main"]);
17539
17540        let mut state = RunState::new(
17541            repo.clone(),
17542            "main".to_owned(),
17543            "deadbeef".to_owned(),
17544            "task".to_owned(),
17545            Config::default(),
17546        );
17547        state.seeds = vec![refs::Seed {
17548            token: "magi/27b2/A".to_owned(),
17549            kind: refs::SeedKind::Unresolved,
17550            sha: String::new(),
17551            branch: true,
17552            detail: "no branch or commit named magi/27b2/A".to_owned(),
17553        }];
17554
17555        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
17556        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
17557        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
17558        let detail = empty_candidate_detail(&state, "main");
17559        assert!(detail.starts_with("empty candidate"), "{detail}");
17560        assert!(!detail.contains("magi/27b2/A"), "{detail}");
17561        assert!(detail.contains("--start-from"), "{detail}");
17562    }
17563
17564    /// `status == Ready` used to be read as "this is the harmless
17565    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
17566    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
17567    /// run's PR was closed without merging — and reentering `merge` with
17568    /// `mode` still `Pr` does not know the difference, so it pushed and
17569    /// opened a second pull request. `mode == Local` reproduces the same
17570    /// blind spot without a network call: reentry must not attempt another
17571    /// git merge once this node has already recorded an outcome.
17572    #[tokio::test]
17573    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
17574        let tmp = tempfile::tempdir().expect("tempdir");
17575        let repo = tmp.path().join("repo");
17576        std::fs::create_dir_all(&repo).unwrap();
17577        init_repo(&repo);
17578
17579        let mut config = Config::default();
17580        config.merge.mode = MergeMode::Local;
17581
17582        let mut state = RunState::new(
17583            repo.clone(),
17584            "main".to_owned(),
17585            "deadbeef".to_owned(),
17586            "task".to_owned(),
17587            config,
17588        );
17589        state.candidates = vec![Candidate {
17590            index: 0,
17591            label: 'A',
17592            agent: "alpha".to_owned(),
17593            branch: "does-not-exist".to_owned(),
17594            worktree: repo.clone(),
17595            summary: String::new(),
17596            stat: String::new(),
17597            files: 0,
17598            commits: 0,
17599            empty: false,
17600            failed: None,
17601            verified_noop: None,
17602            duration_ms: 0,
17603            folded: false,
17604        }];
17605        state.tally = Some(Tally {
17606            first_choice: BTreeMap::from([('A', 1)]),
17607            borda: BTreeMap::new(),
17608            winner: 'A',
17609            rankings: 1,
17610            unanimous_initial: true,
17611            deliberated: false,
17612            changed_votes: 0,
17613            unanimous_final: true,
17614            tie_break: None,
17615            judges: 0,
17616            present: 0,
17617            quorum: 0,
17618            met_quorum: true,
17619            uncontested: Some("only candidate A produced a change".to_owned()),
17620        });
17621        state.reviews = vec![ReviewRound {
17622            round: 1,
17623            head: "deadbeef".to_owned(),
17624            verified_head: None,
17625            verified_at: None,
17626            reviews: Vec::new(),
17627            e2e: Vec::new(),
17628            fix: None,
17629            blocking: 0,
17630            answered: 0,
17631            expected: 0,
17632            clean: true,
17633            verify_retried: false,
17634            e2e_deferred: false,
17635            e2e_defer_reason: None,
17636            progressed: false,
17637            vote_split: false,
17638            reconsideration: Vec::new(),
17639            verdict: None,
17640        }];
17641        state.gate = vec![CommandOutcome {
17642            command: "test".to_owned(),
17643            code: Some(0),
17644            output_tail: String::new(),
17645            duration_ms: 0,
17646            resource_blocked: false,
17647        }];
17648        state.gate_ran = true;
17649        // Reached its conclusion already — e.g. `land` closing the PR without
17650        // merging it, which (like the honest `MergeMode::None` path) leaves
17651        // `status` at `Ready`. The recorded outcome is what actually marks
17652        // this node done.
17653        state.status = RunStatus::Ready;
17654        state.merge = Some(MergeOutcome {
17655            mode: MergeMode::Local,
17656            ok: false,
17657            detail: "already concluded".to_owned(),
17658            empty: false,
17659        });
17660
17661        let mut runner = Runner {
17662            state,
17663            roles: ResolvedRoles {
17664                implementers: Vec::new(),
17665                judges: Vec::new(),
17666                reviewers: Vec::new(),
17667                fixer: None,
17668                conductor: conductor(),
17669                implementer_roster: Vec::new(),
17670                judge_roster: Vec::new(),
17671                reviewer_roster: Vec::new(),
17672            },
17673            sem: Arc::new(Semaphore::new(1)),
17674            pause: Pause::new(),
17675            interrupt: Pause::new(),
17676        };
17677
17678        runner.merge().await.expect("merge");
17679
17680        assert_eq!(
17681            runner.state.status,
17682            RunStatus::Ready,
17683            "a concluded run's status must not change on reentry"
17684        );
17685        assert_eq!(
17686            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17687            Some("already concluded"),
17688            "merge must not run again once the node already recorded an outcome"
17689        );
17690    }
17691
17692    /// `gate` leaves `state.gate_ran` false both before it has ever run and
17693    /// when its last attempt was resource-blocked (the shared build cache
17694    /// could not be acquired or confirmed fresh in time - see
17695    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
17696    /// `Vec` this also leaves behind used to read as "nothing failed" and let
17697    /// a run merge a tree the gate never actually checked - exactly the case
17698    /// a contended cache produces on every retry until it clears. `merge`
17699    /// must refuse until `gate` has actually recorded an attempt.
17700    #[tokio::test]
17701    async fn merge_refuses_a_gate_that_has_not_actually_run() {
17702        let tmp = tempfile::tempdir().expect("tempdir");
17703        let repo = tmp.path().join("repo");
17704        std::fs::create_dir_all(&repo).unwrap();
17705        init_repo(&repo);
17706
17707        let mut config = Config::default();
17708        config.merge.mode = MergeMode::Local;
17709
17710        let mut state = RunState::new(
17711            repo.clone(),
17712            "main".to_owned(),
17713            "deadbeef".to_owned(),
17714            "task".to_owned(),
17715            config,
17716        );
17717        state.candidates = vec![Candidate {
17718            index: 0,
17719            label: 'A',
17720            agent: "alpha".to_owned(),
17721            branch: "does-not-exist".to_owned(),
17722            worktree: repo.clone(),
17723            summary: String::new(),
17724            stat: String::new(),
17725            files: 0,
17726            commits: 0,
17727            empty: false,
17728            failed: None,
17729            verified_noop: None,
17730            duration_ms: 0,
17731            folded: false,
17732        }];
17733        state.tally = Some(Tally {
17734            first_choice: BTreeMap::from([('A', 1)]),
17735            borda: BTreeMap::new(),
17736            winner: 'A',
17737            rankings: 1,
17738            unanimous_initial: true,
17739            deliberated: false,
17740            changed_votes: 0,
17741            unanimous_final: true,
17742            tie_break: None,
17743            judges: 0,
17744            present: 0,
17745            quorum: 0,
17746            met_quorum: true,
17747            uncontested: Some("only candidate A produced a change".to_owned()),
17748        });
17749        state.reviews = vec![ReviewRound {
17750            round: 1,
17751            head: "deadbeef".to_owned(),
17752            verified_head: None,
17753            verified_at: None,
17754            reviews: Vec::new(),
17755            e2e: Vec::new(),
17756            fix: None,
17757            blocking: 0,
17758            answered: 0,
17759            expected: 0,
17760            clean: true,
17761            verify_retried: false,
17762            e2e_deferred: false,
17763            e2e_defer_reason: None,
17764            progressed: false,
17765            vote_split: false,
17766            reconsideration: Vec::new(),
17767            verdict: None,
17768        }];
17769        // The point: `gate` has not recorded anything yet.
17770        state.gate = Vec::new();
17771        state.gate_ran = false;
17772        state.status = RunStatus::Gating;
17773
17774        let mut runner = Runner {
17775            state,
17776            roles: ResolvedRoles {
17777                implementers: Vec::new(),
17778                judges: Vec::new(),
17779                reviewers: Vec::new(),
17780                fixer: None,
17781                conductor: conductor(),
17782                implementer_roster: Vec::new(),
17783                judge_roster: Vec::new(),
17784                reviewer_roster: Vec::new(),
17785            },
17786            sem: Arc::new(Semaphore::new(1)),
17787            pause: Pause::new(),
17788            interrupt: Pause::new(),
17789        };
17790
17791        runner.merge().await.expect("merge");
17792
17793        assert!(
17794            runner.state.merge.is_none(),
17795            "an empty gate must never be read as a passing one: {:?}",
17796            runner.state.merge
17797        );
17798    }
17799
17800    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
17801    /// commands configured and `merge.mode` is `none` (a review-only run).
17802    /// `gate` must still record a real attempt — zero commands, vacuously
17803    /// passed — rather than leaving `state.gate` empty in a way `merge`
17804    /// cannot tell apart from "never ran"; otherwise the run reaches
17805    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
17806    #[tokio::test]
17807    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
17808        let tmp = tempfile::tempdir().expect("tempdir");
17809        let repo = tmp.path().join("repo");
17810        std::fs::create_dir_all(&repo).unwrap();
17811        init_repo(&repo);
17812
17813        // Default config: `verify.gate` empty, `merge.mode` is `none`.
17814        let config = Config::default();
17815
17816        let mut state = RunState::new(
17817            repo.clone(),
17818            "main".to_owned(),
17819            "deadbeef".to_owned(),
17820            "task".to_owned(),
17821            config,
17822        );
17823        state.candidates = vec![Candidate {
17824            index: 0,
17825            label: 'A',
17826            agent: "alpha".to_owned(),
17827            branch: "does-not-exist".to_owned(),
17828            worktree: repo.clone(),
17829            summary: String::new(),
17830            stat: String::new(),
17831            files: 0,
17832            commits: 0,
17833            empty: false,
17834            failed: None,
17835            verified_noop: None,
17836            duration_ms: 0,
17837            folded: false,
17838        }];
17839        state.tally = Some(Tally {
17840            first_choice: BTreeMap::from([('A', 1)]),
17841            borda: BTreeMap::new(),
17842            winner: 'A',
17843            rankings: 1,
17844            unanimous_initial: true,
17845            deliberated: false,
17846            changed_votes: 0,
17847            unanimous_final: true,
17848            tie_break: None,
17849            judges: 0,
17850            present: 0,
17851            quorum: 0,
17852            met_quorum: true,
17853            uncontested: Some("only candidate A produced a change".to_owned()),
17854        });
17855        state.reviews = vec![ReviewRound {
17856            round: 1,
17857            head: "deadbeef".to_owned(),
17858            verified_head: None,
17859            verified_at: None,
17860            reviews: Vec::new(),
17861            e2e: Vec::new(),
17862            fix: None,
17863            blocking: 0,
17864            answered: 0,
17865            expected: 0,
17866            clean: true,
17867            verify_retried: false,
17868            e2e_deferred: false,
17869            e2e_defer_reason: None,
17870            progressed: false,
17871            vote_split: false,
17872            reconsideration: Vec::new(),
17873            verdict: None,
17874        }];
17875
17876        let mut runner = Runner {
17877            state,
17878            roles: ResolvedRoles {
17879                implementers: Vec::new(),
17880                judges: Vec::new(),
17881                reviewers: Vec::new(),
17882                fixer: None,
17883                conductor: conductor(),
17884                implementer_roster: Vec::new(),
17885                judge_roster: Vec::new(),
17886                reviewer_roster: Vec::new(),
17887            },
17888            sem: Arc::new(Semaphore::new(1)),
17889            pause: Pause::new(),
17890            interrupt: Pause::new(),
17891        };
17892
17893        runner.gate().await.expect("gate");
17894        assert!(
17895            runner.state.gate_ran,
17896            "zero configured commands is still a real attempt, not an unrun gate"
17897        );
17898        assert!(runner.state.gate.is_empty());
17899        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
17900        assert_ne!(
17901            runner.state.status,
17902            RunStatus::Blocked,
17903            "a gate with nothing to check must not read as failed"
17904        );
17905
17906        runner.merge().await.expect("merge");
17907        assert_eq!(
17908            runner.state.status,
17909            RunStatus::Ready,
17910            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
17911        );
17912    }
17913
17914    /// `Config::cache_dir` is derived from `verify.e2e` as well as
17915    /// `verify.gate` (so the e2e leg and the final gate never build against
17916    /// different directories). With zero `verify.gate` commands but a
17917    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
17918    /// that lease before discovering it had nothing to run - so a repo with
17919    /// no gate commands could come back `resource_blocked` (and therefore
17920    /// still `gate_ran == false`) on nothing but an unrelated run holding the
17921    /// cache, exactly the contention this run's own zero commands could
17922    /// never have touched. `gate` must recognise there is nothing to check
17923    /// before it ever asks for the lease.
17924    #[tokio::test]
17925    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
17926        crate::run::pin_test_home();
17927        let home = crate::run::home();
17928
17929        let tmp = tempfile::tempdir().expect("tempdir");
17930        let repo = tmp.path().join("repo");
17931        std::fs::create_dir_all(&repo).unwrap();
17932        init_repo(&repo);
17933        // Unique to this test, so holding its lease cannot collide with
17934        // another test sharing the same process-wide `home`.
17935        let cache_dir = tmp.path().join("target");
17936
17937        let mut config = Config::default();
17938        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
17939        // `verify.gate` stays empty (the default). Bounded so a regression
17940        // that does start waiting fails the test in seconds, not hangs it.
17941        config.graph.timeout_verify = Some(2);
17942        config.graph.timeout_resource_wait = 2;
17943
17944        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17945        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17946            .expect("no io error acquiring directly")
17947        {
17948            crate::cache::AcquireOutcome::Acquired(g) => g,
17949            crate::cache::AcquireOutcome::Busy(b) => {
17950                panic!("expected the direct acquire to win the lease first: {b:?}")
17951            }
17952        };
17953
17954        let mut state = RunState::new(
17955            repo.clone(),
17956            "main".to_owned(),
17957            "deadbeef".to_owned(),
17958            "task".to_owned(),
17959            config,
17960        );
17961        state.candidates = vec![Candidate {
17962            index: 0,
17963            label: 'A',
17964            agent: "alpha".to_owned(),
17965            branch: "does-not-exist".to_owned(),
17966            worktree: repo.clone(),
17967            summary: String::new(),
17968            stat: String::new(),
17969            files: 0,
17970            commits: 0,
17971            empty: false,
17972            failed: None,
17973            verified_noop: None,
17974            duration_ms: 0,
17975            folded: false,
17976        }];
17977        state.tally = Some(Tally {
17978            first_choice: BTreeMap::from([('A', 1)]),
17979            borda: BTreeMap::new(),
17980            winner: 'A',
17981            rankings: 1,
17982            unanimous_initial: true,
17983            deliberated: false,
17984            changed_votes: 0,
17985            unanimous_final: true,
17986            tie_break: None,
17987            judges: 0,
17988            present: 0,
17989            quorum: 0,
17990            met_quorum: true,
17991            uncontested: Some("only candidate A produced a change".to_owned()),
17992        });
17993        state.reviews = vec![ReviewRound {
17994            round: 1,
17995            head: "deadbeef".to_owned(),
17996            verified_head: None,
17997            verified_at: None,
17998            reviews: Vec::new(),
17999            e2e: Vec::new(),
18000            fix: None,
18001            blocking: 0,
18002            answered: 0,
18003            expected: 0,
18004            clean: true,
18005            verify_retried: false,
18006            e2e_deferred: false,
18007            e2e_defer_reason: None,
18008            progressed: false,
18009            vote_split: false,
18010            reconsideration: Vec::new(),
18011            verdict: None,
18012        }];
18013
18014        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
18015        state.reviews[0].e2e = vec![CommandOutcome {
18016            command: state.config.verify.e2e[0].clone(),
18017            code: Some(0),
18018            output_tail: String::new(),
18019            duration_ms: 0,
18020            resource_blocked: false,
18021        }];
18022        let mut runner = Runner {
18023            state,
18024            roles: ResolvedRoles {
18025                implementers: Vec::new(),
18026                judges: Vec::new(),
18027                reviewers: Vec::new(),
18028                fixer: None,
18029                conductor: conductor(),
18030                implementer_roster: Vec::new(),
18031                judge_roster: Vec::new(),
18032                reviewer_roster: Vec::new(),
18033            },
18034            sem: Arc::new(Semaphore::new(1)),
18035            pause: Pause::new(),
18036            interrupt: Pause::new(),
18037        };
18038
18039        let started = std::time::Instant::now();
18040        runner.gate().await.expect("gate");
18041        assert!(
18042            started.elapsed() < Duration::from_secs(1),
18043            "a gate with nothing to run must never wait on a lease it never needed"
18044        );
18045        assert!(
18046            runner.state.gate_ran,
18047            "zero commands is still a real, immediate attempt"
18048        );
18049        assert!(runner.state.gate.is_empty());
18050        assert_ne!(
18051            runner.state.status,
18052            RunStatus::Blocked,
18053            "must not read as resource-blocked on a lease it never asked for"
18054        );
18055    }
18056
18057    /// A separate process carries a live worker's environment throughout all
18058    /// verification entry points; the parallel harness never changes its env.
18059    #[cfg(unix)]
18060    #[tokio::test]
18061    async fn verification_is_standalone_inside_a_live_worker() {
18062        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
18063        if std::env::var_os(CHILD).is_none() {
18064            let home = tempfile::tempdir().unwrap();
18065            let generation = crate::worker::pin_current(home.path()).unwrap();
18066            let output = std::process::Command::new(std::env::current_exe().unwrap())
18067                .args([
18068                    "--exact",
18069                    "graph::tests::verification_is_standalone_inside_a_live_worker",
18070                    "--nocapture",
18071                ])
18072                .env(CHILD, "1")
18073                .env("MAGI_HOME", home.path())
18074                .env("MAGI_WORKER_HOME", home.path())
18075                .env("MAGI_WORKER_ID", "abcdef")
18076                .env("MAGI_WORKER_GENERATION", &generation.id)
18077                .env("MAGI_WORKER_EXE", &generation.executable)
18078                .env("MAGI_RUN", "verification-parent")
18079                .env("MAGI_NODE", "gate")
18080                .env("MAGI_SEAT", "worker-fixture")
18081                .env("MAGI_PROCESS_OWNER", "fixture-owner")
18082                .env("MAGI_NO_AUTOUPDATE", "0")
18083                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
18084                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
18085                .env(
18086                    "VERIFY_EXPECT_TOOLCHAIN",
18087                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
18088                )
18089                .env(
18090                    "VERIFY_EXPECT_CACHE",
18091                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
18092                )
18093                .output()
18094                .unwrap();
18095            assert!(
18096                output.status.success()
18097                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
18098                "{}\n{}",
18099                String::from_utf8_lossy(&output.stdout),
18100                String::from_utf8_lossy(&output.stderr)
18101            );
18102            return;
18103        }
18104        let home = crate::run::home();
18105        let directory = home.join("workers/jobs/abcdef");
18106        std::fs::create_dir_all(&directory).unwrap();
18107        let job = crate::worker::Job {
18108            id: "abcdef".into(),
18109            sequence: 1,
18110            kind: "task".into(),
18111            key: "task:verification".into(),
18112            payload: serde_json::json!({}),
18113            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
18114            version: Some(env!("CARGO_PKG_VERSION").into()),
18115            status: crate::worker::Status::Running,
18116            result: None,
18117            pid: Some(std::process::id()),
18118            created_at: 0,
18119            finished_at: None,
18120        };
18121        std::fs::write(
18122            directory.join("job.json"),
18123            serde_json::to_vec(&job).unwrap(),
18124        )
18125        .unwrap();
18126        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
18127        let assert_worker = || {
18128            assert_eq!(
18129                crate::worker::current_job(&home).unwrap().unwrap().id,
18130                job.id
18131            );
18132            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
18133            assert_eq!(
18134                crate::worker::helper_executable(&home).unwrap(),
18135                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
18136            );
18137            assert!(
18138                crate::updater::Checker::new(&crate::config::Update {
18139                    mode: crate::config::UpdateMode::Notify,
18140                    interval: None,
18141                    auto_restart: false,
18142                })
18143                .is_none(),
18144                "the driver remains pinned, even while tests are standalone"
18145            );
18146        };
18147        assert_worker();
18148        let repo = home.join("repo");
18149        std::fs::create_dir_all(&repo).unwrap();
18150        init_repo(&repo);
18151        // Keep storage, seat/control context, containment and tool lookup.
18152        // No inherited values (or credentials) are printed on success.
18153        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
18154            test -z "${MAGI_WORKER_GENERATION+x}" &&
18155            test -z "${MAGI_WORKER_HOME+x}" &&
18156            test -z "${MAGI_WORKER_EXE+x}" &&
18157            test "$MAGI_HOME/repo" -ef . &&
18158            test "$MAGI_RUN" = verification-parent &&
18159            test "$MAGI_NODE" = gate &&
18160            test "$MAGI_SEAT" = worker-fixture &&
18161            test -n "$MAGI_PROCESS_OWNER" &&
18162            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
18163            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
18164            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
18165            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
18166        // Negative control: an ordinary child inherits the contamination.
18167        assert!(
18168            !std::process::Command::new("sh")
18169                .args(["-c", probe])
18170                .current_dir(&repo)
18171                .status()
18172                .unwrap()
18173                .success()
18174        );
18175        let unrelated_home = tempfile::tempdir().unwrap();
18176        assert!(
18177            crate::worker::current_job(unrelated_home.path()).is_err(),
18178            "an inherited worker id cannot be resolved in a unit fixture's home"
18179        );
18180        let mut runner = runner_at(RunStatus::Gating);
18181        runner.state.repo = repo.clone();
18182        runner.state.config.disk.min_free_bytes = 0;
18183        runner.state.config.coordination = crate::config::test_coordination();
18184        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
18185        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
18186        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
18187        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
18188        let winner = Candidate {
18189            index: 0,
18190            label: 'A',
18191            agent: "fixture".into(),
18192            branch: "main".into(),
18193            worktree: repo.clone(),
18194            summary: String::new(),
18195            stat: String::new(),
18196            files: 0,
18197            commits: 0,
18198            empty: false,
18199            failed: None,
18200            verified_noop: None,
18201            duration_ms: 0,
18202            folded: false,
18203        };
18204        let (e2e, retried, pids) = run_e2e_with_retry(
18205            &mut runner.state,
18206            &["sh".into(), "-c".into()],
18207            &[format!("{probe} && printf initial")],
18208            &repo,
18209            Duration::from_secs(10),
18210            "initial",
18211        )
18212        .await;
18213        assert!(!retried && pids.is_empty());
18214        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
18215        assert_eq!(e2e[0].output_tail, "initial");
18216        let gate = runner.run_gate(&winner).await.unwrap();
18217        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
18218        // An interrupted repair must recheck e2e from saved state. Landing
18219        // reuses that state and must run both pre-gate and gate again.
18220        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
18221            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
18222            passed: false,
18223            outcomes: Vec::new(),
18224        });
18225        runner.state.save().unwrap();
18226        let mut resumed = RunState::load(&runner.state.id).unwrap();
18227        let wait = crate::repo_lock::WaitPolicy::default();
18228        assert!(matches!(
18229            check_gate_repair_e2e(&mut resumed, &winner, &wait)
18230                .await
18231                .unwrap(),
18232            GateFix::Retry
18233        ));
18234        let saved = RunState::load(&resumed.id).unwrap();
18235        let repair = saved.gate_repair_e2e.unwrap();
18236        assert!(repair.passed);
18237        assert_eq!(repair.outcomes.len(), 1);
18238        assert!(repair.outcomes[0].ok());
18239        let landing = run_landing_gate(&mut resumed, &winner, &wait)
18240            .await
18241            .unwrap();
18242        assert_eq!(landing.len(), 1);
18243        assert!(landing[0].ok(), "{landing:?}");
18244        assert_eq!(
18245            std::fs::read_to_string(repo.join("phases")).unwrap(),
18246            "pre\ngate\ne2e\npre\ngate\n"
18247        );
18248        assert_worker();
18249    }
18250
18251    #[cfg(unix)]
18252    #[tokio::test]
18253    async fn verification_keeps_the_panic_and_the_full_output() {
18254        crate::run::pin_test_home();
18255        let tmp = tempfile::tempdir().unwrap();
18256        let mut config = Config::default();
18257        config.disk.min_free_bytes = 0;
18258        let mut state = RunState::new(
18259            tmp.path().to_owned(),
18260            "main".into(),
18261            "base".into(),
18262            "verification evidence".into(),
18263            config,
18264        );
18265        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
18266                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
18267                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
18268        let mut names = Vec::new();
18269        for _ in 0..2 {
18270            let (outcomes, _) = run_commands(
18271                &mut state,
18272                "gate",
18273                "gate",
18274                0,
18275                &["sh".into(), "-c".into()],
18276                &[script.into()],
18277                tmp.path(),
18278                Duration::from_secs(60),
18279            )
18280            .await;
18281            let tail = &outcomes[0].output_tail;
18282            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
18283            assert!(tail.contains("boom assertion"));
18284            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
18285            let line = tail.lines().next().unwrap();
18286            names.push(line.to_owned());
18287        }
18288        assert_ne!(names[0], names[1], "the second run must not overwrite");
18289        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
18290        assert!(full.len() > 500_000 && full.contains("boom assertion"));
18291        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
18292
18293        // Storage failures are evidence too; preserve the command's verdict.
18294        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
18295        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
18296        let (outcomes, _) = run_commands(
18297            &mut state,
18298            "gate",
18299            "gate",
18300            0,
18301            &["sh".into(), "-c".into()],
18302            &["echo failure-output; exit 7".into()],
18303            tmp.path(),
18304            Duration::from_secs(10),
18305        )
18306        .await;
18307        assert_eq!(outcomes[0].code, Some(7));
18308        assert!(state.events.iter().any(|e| {
18309            e.node == "gate" && e.message.contains("could not save the full gate output:")
18310        }));
18311    }
18312
18313    #[cfg(unix)]
18314    #[tokio::test]
18315    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
18316        crate::run::pin_test_home();
18317        let tmp = tempfile::tempdir().unwrap();
18318        let mut config = Config::default();
18319        config.disk.min_free_bytes = 0;
18320        let mut state = RunState::new(
18321            tmp.path().to_owned(),
18322            "main".into(),
18323            "base".into(),
18324            "verification evidence".into(),
18325            config,
18326        );
18327        let (outcomes, pids) = run_commands(
18328            &mut state,
18329            "gate",
18330            "gate",
18331            1,
18332            &["sh".into(), "-c".into()],
18333            &[
18334                "printf before-timeout; exec sleep 30".into(),
18335                "kill -TERM $$".into(),
18336                "exit 7".into(),
18337                "printf success".into(),
18338            ],
18339            tmp.path(),
18340            Duration::from_secs(1),
18341        )
18342        .await;
18343        wait_for_timed_out_children_to_die(&pids).await;
18344        assert_eq!(
18345            pids.len(),
18346            1,
18347            "unleased timeout retains its death-check PID"
18348        );
18349        assert_eq!(outcomes.len(), 4);
18350        assert_eq!(outcomes[0].code, None);
18351        assert!(outcomes[0].output_tail.contains("before-timeout"));
18352        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
18353        assert_eq!(outcomes[1].code, None);
18354        assert!(outcomes[1].output_tail.contains("command terminated:"));
18355        assert!(!outcomes[1].output_tail.contains("timed out"));
18356        assert_eq!(outcomes[2].code, Some(7));
18357        assert!(!outcomes[2].output_tail.contains("timed out"));
18358        assert_eq!(outcomes[3].code, Some(0));
18359        assert_eq!(outcomes[3].output_tail, "success");
18360        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
18361    }
18362
18363    /// The addendum's second gap: a `verify.gate` command running for real
18364    /// wall-clock time had nothing at all to show for it in `active` before
18365    /// `run_commands` learned to record it — a run could sit in `Gating` for
18366    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
18367    /// what was actually happening. Proven with a genuinely still-running
18368    /// command, not just a before/after check on the final state: a poller
18369    /// task reads the same `run.json` `gate()` is writing, the same way the
18370    /// phone or `magi show` would, while the shell command is still blocked
18371    /// on its own release marker.
18372    #[tokio::test]
18373    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
18374        crate::run::pin_test_home();
18375
18376        let tmp = tempfile::tempdir().expect("tempdir");
18377        let repo = tmp.path().join("repo");
18378        std::fs::create_dir_all(&repo).unwrap();
18379        init_repo(&repo);
18380
18381        let mut config = Config::default();
18382        config.verify.gate = vec![
18383            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
18384             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
18385                .to_owned(),
18386        ];
18387
18388        let mut state = RunState::new(
18389            repo.clone(),
18390            "main".to_owned(),
18391            "deadbeef".to_owned(),
18392            "task".to_owned(),
18393            config,
18394        );
18395        let run_id = state.id.clone();
18396        state.candidates = vec![Candidate {
18397            index: 0,
18398            label: 'A',
18399            agent: "alpha".to_owned(),
18400            branch: "does-not-exist".to_owned(),
18401            worktree: repo.clone(),
18402            summary: String::new(),
18403            stat: String::new(),
18404            files: 0,
18405            commits: 0,
18406            empty: false,
18407            failed: None,
18408            verified_noop: None,
18409            duration_ms: 0,
18410            folded: false,
18411        }];
18412        state.tally = Some(Tally {
18413            first_choice: BTreeMap::from([('A', 1)]),
18414            borda: BTreeMap::new(),
18415            winner: 'A',
18416            rankings: 1,
18417            unanimous_initial: true,
18418            deliberated: false,
18419            changed_votes: 0,
18420            unanimous_final: true,
18421            tie_break: None,
18422            judges: 0,
18423            present: 0,
18424            quorum: 0,
18425            met_quorum: true,
18426            uncontested: Some("only candidate A produced a change".to_owned()),
18427        });
18428        state.reviews = vec![ReviewRound {
18429            round: 1,
18430            head: "deadbeef".to_owned(),
18431            verified_head: None,
18432            verified_at: None,
18433            reviews: Vec::new(),
18434            e2e: Vec::new(),
18435            fix: None,
18436            blocking: 0,
18437            answered: 0,
18438            expected: 0,
18439            clean: true,
18440            verify_retried: false,
18441            e2e_deferred: false,
18442            e2e_defer_reason: None,
18443            progressed: false,
18444            vote_split: false,
18445            reconsideration: Vec::new(),
18446            verdict: None,
18447        }];
18448
18449        let mut runner = Runner {
18450            state,
18451            roles: ResolvedRoles {
18452                implementers: Vec::new(),
18453                judges: Vec::new(),
18454                reviewers: Vec::new(),
18455                fixer: None,
18456                conductor: conductor(),
18457                implementer_roster: Vec::new(),
18458                judge_roster: Vec::new(),
18459                reviewer_roster: Vec::new(),
18460            },
18461            sem: Arc::new(Semaphore::new(1)),
18462            pause: Pause::new(),
18463            interrupt: Pause::new(),
18464        };
18465
18466        let started_marker = repo.join("started.marker");
18467        let release_marker = repo.join("release.marker");
18468        let poller = tokio::spawn(async move {
18469            // Bounded so a regression that never records the task entry
18470            // fails this test in seconds instead of hanging the suite —
18471            // the same shape `a_park_requested_while_a_seat_is_mid_call_
18472            // does_not_cut_it_short` uses for the same reason.
18473            for _ in 0..100 {
18474                if started_marker.exists()
18475                    && let Ok(s) = crate::run::RunState::load(&run_id)
18476                    && let Some(a) = s.active.get("gate")
18477                {
18478                    std::fs::write(&release_marker, b"go").expect("release marker");
18479                    return Some(a.clone());
18480                }
18481                tokio::time::sleep(Duration::from_millis(50)).await;
18482            }
18483            None
18484        });
18485
18486        runner.gate().await.expect("gate");
18487        let captured = poller.await.expect("poller task");
18488        let captured = captured.expect(
18489            "the poller never saw a `gate` task entry in run.json while the command was \
18490             still blocked on its own release marker",
18491        );
18492
18493        assert_eq!(captured.task.as_deref(), Some("gate"));
18494        assert_eq!(captured.node, "gate");
18495        assert_eq!(captured.index, Some(1));
18496        assert_eq!(captured.total, Some(1));
18497        assert!(
18498            captured
18499                .command
18500                .as_deref()
18501                .is_some_and(|c| c.contains("started.marker")),
18502            "{captured:?}"
18503        );
18504
18505        assert!(
18506            runner.state.active.is_empty(),
18507            "the entry must be cleared once the command actually finished: {:?}",
18508            runner.state.active
18509        );
18510        assert!(runner.state.gate_ran);
18511        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18512    }
18513
18514    /// The hand-off over a blocking finding a reviewer rejected on leaves a
18515    /// record for `land`; one with only a Minor, or no reject, leaves none.
18516    #[tokio::test]
18517    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
18518        use crate::verdict::{Finding, ReviewVote, Severity};
18519        crate::run::pin_test_home();
18520        let tmp = tempfile::tempdir().expect("tempdir");
18521        let repo = tmp.path().join("repo");
18522        std::fs::create_dir_all(&repo).unwrap();
18523        init_repo(&repo);
18524
18525        for (severity, vote, expect) in [
18526            (Severity::Major, ReviewVote::Reject, true),
18527            (Severity::Minor, ReviewVote::Reject, false),
18528            (Severity::Major, ReviewVote::Approve, false),
18529        ] {
18530            let mut round = review_round(false, 1, 1, 1, false, true);
18531            round.reviews = vec![ReviewRecord {
18532                reviewer: 1,
18533                agent: "alpha".to_owned(),
18534                summary: String::new(),
18535                findings: vec![Finding {
18536                    id: "R1-1-1".to_owned(),
18537                    severity,
18538                    file: None,
18539                    line: None,
18540                    title: "t".to_owned(),
18541                    detail: String::new(),
18542                }],
18543                vote: Some(vote),
18544                failed: None,
18545                duration_ms: 0,
18546                attempts: 0,
18547            }];
18548            let mut state = RunState::new(
18549                repo.clone(),
18550                "main".to_owned(),
18551                "deadbeef".to_owned(),
18552                "task".to_owned(),
18553                Config::default(),
18554            );
18555            state.reviews = vec![round];
18556            let mut runner = Runner {
18557                state,
18558                roles: ResolvedRoles {
18559                    implementers: Vec::new(),
18560                    judges: Vec::new(),
18561                    reviewers: Vec::new(),
18562                    fixer: None,
18563                    conductor: conductor(),
18564                    implementer_roster: Vec::new(),
18565                    judge_roster: Vec::new(),
18566                    reviewer_roster: Vec::new(),
18567                },
18568                sem: Arc::new(Semaphore::new(1)),
18569                pause: Pause::new(),
18570                interrupt: Pause::new(),
18571            };
18572            let shell = runner.state.config.shell();
18573            runner
18574                .stop_reviewing("round budget spent", &shell, &repo)
18575                .await
18576                .expect("stop_reviewing");
18577            assert_eq!(runner.state.status, RunStatus::Gating);
18578            assert_eq!(
18579                runner.state.contested_handoff.is_some(),
18580                expect,
18581                "{severity:?} + {vote:?}"
18582            );
18583        }
18584    }
18585
18586    /// The shape the incident this whole fix responds to actually had: the
18587    /// round budget spent, the last round's own e2e blocked on the shared
18588    /// build cache (held here by a live pid — this test process — exactly
18589    /// `cache`'s own unit tests' pattern for "another owner, still alive"
18590    /// without forking a process). `stop_reviewing` must retry it — not
18591    /// silently leave the round looking untouched (the catch-up-only half of
18592    /// the bug), and not read the contention as a red `e2e` and block the
18593    /// run on it (the other half). Called directly, the same way
18594    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
18595    /// above exercises `gate`, so this never needs a real cargo build to
18596    /// reach: the lease is never released, so `with_cache_lease` never gets
18597    /// past acquiring it into anything that would need a real workspace.
18598    #[tokio::test]
18599    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
18600        crate::run::pin_test_home();
18601        let home = crate::run::home();
18602
18603        let tmp = tempfile::tempdir().expect("tempdir");
18604        let repo = tmp.path().join("repo");
18605        std::fs::create_dir_all(&repo).unwrap();
18606        init_repo(&repo);
18607        let head = crate::git::rev_parse(&repo, "HEAD")
18608            .await
18609            .expect("rev-parse");
18610        // Unique to this test, so holding its lease cannot collide with
18611        // another test sharing the same process-wide `home`.
18612        let cache_dir = tmp.path().join("target");
18613
18614        let mut config = Config::default();
18615        config.verify.e2e = vec![format!(
18616            "CARGO_TARGET_DIR='{}' test -f README.md",
18617            cache_dir.display()
18618        )];
18619        config.graph.review_rounds = 1;
18620        // Bounded so a regression that does start waiting fails the test in
18621        // seconds, not hangs it.
18622        config.graph.timeout_verify = Some(2);
18623        config.graph.timeout_resource_wait = 2;
18624
18625        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18626        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18627            .expect("no io error acquiring directly")
18628        {
18629            crate::cache::AcquireOutcome::Acquired(g) => g,
18630            crate::cache::AcquireOutcome::Busy(b) => {
18631                panic!("expected the direct acquire to win the lease first: {b:?}")
18632            }
18633        };
18634
18635        let mut state = RunState::new(
18636            repo.clone(),
18637            "main".to_owned(),
18638            head.clone(),
18639            "task".to_owned(),
18640            config,
18641        );
18642        state.candidates = vec![Candidate {
18643            index: 0,
18644            label: 'A',
18645            agent: "alpha".to_owned(),
18646            branch: "does-not-exist".to_owned(),
18647            worktree: repo.clone(),
18648            summary: String::new(),
18649            stat: String::new(),
18650            files: 0,
18651            commits: 0,
18652            empty: false,
18653            failed: None,
18654            verified_noop: None,
18655            duration_ms: 0,
18656            folded: false,
18657        }];
18658        state.tally = Some(Tally {
18659            first_choice: BTreeMap::from([('A', 1)]),
18660            borda: BTreeMap::new(),
18661            winner: 'A',
18662            rankings: 1,
18663            unanimous_initial: true,
18664            deliberated: false,
18665            changed_votes: 0,
18666            unanimous_final: true,
18667            tie_break: None,
18668            judges: 0,
18669            present: 0,
18670            quorum: 0,
18671            met_quorum: true,
18672            uncontested: Some("only candidate A produced a change".to_owned()),
18673        });
18674        // The round budget's last round, deferred: `needs_catchup_run`'s
18675        // other trigger. `stop_reviewing`'s retry machinery must treat this
18676        // exactly like a resource-blocked attempt once it actually runs.
18677        state.reviews = vec![ReviewRound {
18678            round: 1,
18679            head: head.clone(),
18680            verified_head: None,
18681            verified_at: None,
18682            reviews: Vec::new(),
18683            e2e: Vec::new(),
18684            fix: None,
18685            blocking: 1,
18686            answered: 1,
18687            expected: 1,
18688            clean: false,
18689            verify_retried: false,
18690            e2e_deferred: true,
18691            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
18692            progressed: false,
18693            vote_split: false,
18694            reconsideration: Vec::new(),
18695            verdict: None,
18696        }];
18697
18698        let mut runner = Runner {
18699            state,
18700            roles: ResolvedRoles {
18701                implementers: Vec::new(),
18702                judges: Vec::new(),
18703                reviewers: Vec::new(),
18704                fixer: None,
18705                conductor: conductor(),
18706                implementer_roster: Vec::new(),
18707                judge_roster: Vec::new(),
18708                reviewer_roster: Vec::new(),
18709            },
18710            sem: Arc::new(Semaphore::new(1)),
18711            pause: Pause::new(),
18712            interrupt: Pause::new(),
18713        };
18714
18715        let shell = runner.state.config.shell();
18716        runner
18717            .stop_reviewing("round budget spent", &shell, &repo)
18718            .await
18719            .expect("stop_reviewing");
18720
18721        let last = runner.state.reviews.last().expect("round record");
18722        assert_eq!(
18723            last.e2e_status(),
18724            E2eStatus::ResourceBlocked,
18725            "the shared cache is still held; the attempt must read as blocked, not deferred or \
18726             failed: {last:?}"
18727        );
18728        assert_eq!(
18729            last.verified_head.as_deref(),
18730            Some(head.as_str()),
18731            "which commit this attempt targeted is known even though nothing finished checking \
18732             it"
18733        );
18734        let first_attempt_at = last
18735            .verified_at
18736            .expect("when this attempt ran is known too");
18737        assert_ne!(
18738            runner.state.status,
18739            RunStatus::Blocked,
18740            "contention is evidence about the machine, not the patch — it must not settle the \
18741             run as blocked: {:?}",
18742            runner.state.status
18743        );
18744        assert!(
18745            !runner
18746                .state
18747                .events
18748                .iter()
18749                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
18750            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
18751            runner.state.events
18752        );
18753
18754        // The cache is still held: a later reentry must retry the same
18755        // round's verification again — not leave it looking exactly as
18756        // untouched as the first blocked attempt, which is indistinguishable
18757        // from never having tried again at all.
18758        runner
18759            .stop_reviewing("round budget spent", &shell, &repo)
18760            .await
18761            .expect("stop_reviewing retry");
18762        assert_eq!(
18763            runner.state.reviews.len(),
18764            1,
18765            "no new round was started: {:?}",
18766            runner.state.reviews
18767        );
18768        let last = runner.state.reviews.last().expect("round record");
18769        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
18770        assert!(
18771            last.verified_at.expect("still known") > first_attempt_at,
18772            "a second reentry must be a fresh attempt, not a stale copy of the first"
18773        );
18774        assert_ne!(runner.state.status, RunStatus::Blocked);
18775
18776        held.release();
18777    }
18778
18779    #[cfg(unix)]
18780    #[tokio::test]
18781    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
18782        ask_test_home();
18783        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
18784            let tmp = tempfile::tempdir().unwrap();
18785            let repo = tmp.path().join("repo");
18786            std::fs::create_dir(&repo).unwrap();
18787            init_repo(&repo);
18788            let run_git = |args: &[&str]| {
18789                let out = std::process::Command::new("git")
18790                    .args(args)
18791                    .current_dir(&repo)
18792                    .output()
18793                    .unwrap();
18794                assert!(
18795                    out.status.success(),
18796                    "{args:?}: {}",
18797                    String::from_utf8_lossy(&out.stderr)
18798                );
18799            };
18800            let origin = tmp.path().join("origin.git");
18801            run_git(&["init", "--bare", origin.to_str().unwrap()]);
18802            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
18803            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
18804            run_git(&["add", "spec.rs"]);
18805            run_git(&["commit", "-m", "initial spec"]);
18806            run_git(&["push", "origin", "main"]);
18807            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
18808            run_git(&["checkout", "-b", "fix/review"]);
18809            std::fs::write(
18810                repo.join("check.rs"),
18811                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
18812            )
18813            .unwrap();
18814            run_git(&["add", "check.rs"]);
18815            run_git(&["commit", "-m", "use spec"]);
18816            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
18817            run_git(&["checkout", "main"]);
18818            std::fs::write(
18819                repo.join("spec.rs"),
18820                "struct AgentSpec { max_turn_secs: u64 }\n",
18821            )
18822            .unwrap();
18823            run_git(&["add", "spec.rs"]);
18824            run_git(&["commit", "-m", "extend spec"]);
18825            run_git(&["push", "origin", "main"]);
18826            run_git(&["checkout", "fix/review"]);
18827
18828            let mut runner = runner_at(RunStatus::Reviewing);
18829            runner.state.repo = repo.clone();
18830            runner.state.base_commit = base;
18831            runner.state.pr_head = Some("fix/review".into());
18832            runner.state.config.graph.review_rounds = max_rounds;
18833            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
18834            runner.state.config.graph.sessions = false;
18835            runner.state.config.disk.min_free_bytes = 0;
18836            runner.state.config.verify.e2e =
18837                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
18838            runner.state.candidates = vec![Candidate {
18839                index: 0,
18840                label: 'A',
18841                agent: "fixture".into(),
18842                branch: "fix/review".into(),
18843                worktree: repo.clone(),
18844                summary: String::new(),
18845                stat: String::new(),
18846                files: 1,
18847                commits: 1,
18848                empty: false,
18849                failed: None,
18850                verified_noop: None,
18851                duration_ms: 0,
18852                folded: false,
18853            }];
18854            runner.state.tally = Some(Tally {
18855                first_choice: BTreeMap::new(),
18856                borda: BTreeMap::new(),
18857                winner: 'A',
18858                rankings: 0,
18859                unanimous_initial: true,
18860                deliberated: false,
18861                changed_votes: 0,
18862                unanimous_final: true,
18863                tie_break: None,
18864                judges: 0,
18865                present: 0,
18866                quorum: 0,
18867                met_quorum: true,
18868                uncontested: Some("review-only".into()),
18869            });
18870            let mut last = review_round(false, 0, 1, 1, false, false);
18871            last.head = reviewed.clone();
18872            last.e2e_deferred = true;
18873            last.e2e[0].resource_blocked = true;
18874            last.e2e[0].output_tail = "waiting for the shared build cache".into();
18875            if max_rounds == 1 {
18876                last.e2e.clear();
18877            }
18878            runner.state.reviews = vec![last];
18879            let calls = tmp.path().join("fix-calls");
18880            let mut fixer = spec("repair");
18881            fixer
18882                .env
18883                .insert("FIX_CALLS".into(), calls.display().to_string());
18884            fixer.command = vec!["sh".into(), "-c".into(), r#"
18885                echo fix >> "$FIX_CALLS"
18886                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
18887                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
18888                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
18889                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
18890            "#.into()];
18891            let mut reviewer = spec("reviewer");
18892            reviewer.command = vec![
18893                "sh".into(),
18894                "-c".into(),
18895                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
18896                    .into(),
18897            ];
18898            runner.roles.fixer = Some(vec![fixer]);
18899            runner.roles.reviewers = vec![reviewer];
18900
18901            runner.sync_to_base().await.unwrap();
18902            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
18903            assert_ne!(rebased, reviewed);
18904            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
18905            if reentry {
18906                // Simulate a crash after catch-up evidence was saved but before
18907                // dispatching the fixer. No dedicated pending-repair field exists.
18908                let shell = runner.state.config.shell();
18909                runner
18910                    .stop_unresolved_review("deferred verification", &shell, &repo)
18911                    .await
18912                    .unwrap();
18913                assert!(runner.catchup_repairable());
18914                assert!(!calls.exists());
18915                runner.state = RunState::load(&runner.state.id).unwrap();
18916                assert!(runner.catchup_repairable());
18917            }
18918            runner.review_loop().await.unwrap();
18919            let first = &runner.state.reviews[0];
18920            assert_eq!(first.head, reviewed);
18921            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
18922            assert!(
18923                first.verify_retried,
18924                "a reproduced build failure gets exactly one retry: {:?}",
18925                first.e2e
18926            );
18927            assert!(first.e2e[0].output_tail.contains("E0063"));
18928            assert!(!runner.catchup_repairable());
18929            if max_rounds == 2 {
18930                assert_eq!(
18931                    runner.state.status,
18932                    RunStatus::Gating,
18933                    "{}",
18934                    crate::report::run(&runner.state)
18935                );
18936                assert_eq!(runner.state.reviews.len(), 2);
18937                assert!(first.fix.as_ref().unwrap().committed);
18938                assert!(runner.state.reviews[1].clean);
18939                assert_eq!(
18940                    runner
18941                        .state
18942                        .reviews
18943                        .iter()
18944                        .filter(|r| r.fix.is_some())
18945                        .count(),
18946                    1
18947                );
18948                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
18949                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
18950            } else {
18951                assert_eq!(runner.state.status, RunStatus::Blocked);
18952                assert!(first.fix.is_none());
18953                assert!(!calls.exists());
18954                let recovery = crate::recovery::assess(
18955                    "prior",
18956                    Some(&runner.state),
18957                    &runner.landing_base(),
18958                    crate::recovery::Relation::Diverged,
18959                );
18960                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
18961                let reasons = recovery.reasons.join("\n");
18962                assert!(
18963                    reasons.contains(&reviewed) && reasons.contains(&rebased),
18964                    "{reasons}"
18965                );
18966                assert!(
18967                    reasons.contains("base sync to")
18968                        && reasons.contains("additional review is required"),
18969                    "{reasons}"
18970                );
18971            }
18972        }
18973    }
18974
18975    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
18976    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
18977    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
18978    /// own top-of-function fast path (`review_conclusion`) correctly reads
18979    /// this shape as `None` rather than guessing `Blocked`, and the loop's
18980    /// own `for` range is empty once the round budget is spent, so
18981    /// `review_loop` must retry the check itself rather than silently doing
18982    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
18983    /// above exercises directly, but through `review_loop`'s own entry point
18984    /// this time, proving the wiring between the two rather than just the
18985    /// retry logic in isolation.
18986    #[tokio::test]
18987    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
18988        crate::run::pin_test_home();
18989        let home = crate::run::home();
18990
18991        let tmp = tempfile::tempdir().expect("tempdir");
18992        let repo = tmp.path().join("repo");
18993        std::fs::create_dir_all(&repo).unwrap();
18994        init_repo(&repo);
18995        let head = crate::git::rev_parse(&repo, "HEAD")
18996            .await
18997            .expect("rev-parse");
18998        let cache_dir = tmp.path().join("target");
18999
19000        let mut config = Config::default();
19001        config.verify.e2e = vec![format!(
19002            "CARGO_TARGET_DIR='{}' test -f README.md",
19003            cache_dir.display()
19004        )];
19005        config.graph.review_rounds = 1;
19006        config.graph.timeout_verify = Some(2);
19007        config.graph.timeout_resource_wait = 2;
19008
19009        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19010        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19011            .expect("no io error acquiring directly")
19012        {
19013            crate::cache::AcquireOutcome::Acquired(g) => g,
19014            crate::cache::AcquireOutcome::Busy(b) => {
19015                panic!("expected the direct acquire to win the lease first: {b:?}")
19016            }
19017        };
19018
19019        let mut state = RunState::new(
19020            repo.clone(),
19021            "main".to_owned(),
19022            head.clone(),
19023            "task".to_owned(),
19024            config,
19025        );
19026        state.candidates = vec![Candidate {
19027            index: 0,
19028            label: 'A',
19029            agent: "alpha".to_owned(),
19030            branch: "does-not-exist".to_owned(),
19031            worktree: repo.clone(),
19032            summary: String::new(),
19033            stat: String::new(),
19034            files: 0,
19035            commits: 0,
19036            empty: false,
19037            failed: None,
19038            verified_noop: None,
19039            duration_ms: 0,
19040            folded: false,
19041        }];
19042        state.tally = Some(Tally {
19043            first_choice: BTreeMap::from([('A', 1)]),
19044            borda: BTreeMap::new(),
19045            winner: 'A',
19046            rankings: 1,
19047            unanimous_initial: true,
19048            deliberated: false,
19049            changed_votes: 0,
19050            unanimous_final: true,
19051            tie_break: None,
19052            judges: 0,
19053            present: 0,
19054            quorum: 0,
19055            met_quorum: true,
19056            uncontested: Some("only candidate A produced a change".to_owned()),
19057        });
19058        // The exact shape a prior process's `stop_reviewing` would have left
19059        // on disk: the round budget's last round, a real attempt already
19060        // made and already resource-blocked.
19061        state.reviews = vec![ReviewRound {
19062            round: 1,
19063            head: head.clone(),
19064            verified_head: Some(head.clone()),
19065            verified_at: Some(jiff::Timestamp::now()),
19066            reviews: Vec::new(),
19067            e2e: vec![CommandOutcome {
19068                command: format!(
19069                    "CARGO_TARGET_DIR='{}' test -f README.md",
19070                    cache_dir.display()
19071                ),
19072                code: None,
19073                output_tail: "waiting for the shared build cache".to_owned(),
19074                duration_ms: 0,
19075                resource_blocked: true,
19076            }],
19077            fix: None,
19078            blocking: 1,
19079            answered: 1,
19080            expected: 1,
19081            clean: false,
19082            verify_retried: false,
19083            e2e_deferred: false,
19084            e2e_defer_reason: None,
19085            progressed: false,
19086            vote_split: false,
19087            reconsideration: Vec::new(),
19088            verdict: None,
19089        }];
19090
19091        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
19092        let mut runner = Runner {
19093            state,
19094            roles: ResolvedRoles {
19095                implementers: Vec::new(),
19096                judges: Vec::new(),
19097                reviewers: Vec::new(),
19098                fixer: None,
19099                conductor: conductor(),
19100                implementer_roster: Vec::new(),
19101                judge_roster: Vec::new(),
19102                reviewer_roster: Vec::new(),
19103            },
19104            sem: Arc::new(Semaphore::new(1)),
19105            pause: Pause::new(),
19106            interrupt: Pause::new(),
19107        };
19108
19109        // The lease is still held throughout, so this reentry's own retry is
19110        // also contended — proving `review_loop` actually tried again (not
19111        // that it happened to succeed) is what the timestamp comparison
19112        // below is for.
19113        runner.review_loop().await.expect("review_loop");
19114
19115        assert_eq!(
19116            runner.state.reviews.len(),
19117            1,
19118            "no new round was started on top of the unresolved one: {:?}",
19119            runner.state.reviews
19120        );
19121        let last = &runner.state.reviews[0];
19122        assert_eq!(
19123            last.e2e_status(),
19124            E2eStatus::ResourceBlocked,
19125            "still contended: {last:?}"
19126        );
19127        assert!(
19128            last.verified_at.expect("still known") > first_attempt_at,
19129            "review_loop must have actually retried the check, not left it exactly as found"
19130        );
19131        assert_ne!(
19132            runner.state.status,
19133            RunStatus::Blocked,
19134            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
19135            runner.state.status
19136        );
19137
19138        held.release();
19139    }
19140
19141    #[tokio::test]
19142    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
19143        crate::run::pin_test_home();
19144        for status in [RunStatus::Blocked, RunStatus::Gating] {
19145            let mut runner = runner_at(status);
19146            runner.state.config.graph.land = true;
19147            runner.state.gate_ran = true;
19148            runner.state.merge = Some(MergeOutcome {
19149                mode: MergeMode::Pr,
19150                ok: false,
19151                detail: "the fixer produced no commit".into(),
19152                empty: false,
19153            });
19154            let url = "https://example.invalid/o/r/pull/1";
19155            runner.state.pr = Some(crate::run::PrRecord {
19156                url: url.into(),
19157                number: 1,
19158                state: "open".into(),
19159                checks: "red".into(),
19160                round: 1,
19161                rounds: 1,
19162                red_at_merge: Vec::new(),
19163            });
19164            runner.state.save().unwrap();
19165            // The nonexistent repository makes observation fail before a forge
19166            // command can run. We still must reach land, rather than silently
19167            // return from the graph with its old gate and merge records.
19168            runner.execute_graph().await.unwrap();
19169            assert_eq!(runner.state.status, RunStatus::Blocked);
19170            assert!(runner.state.land_sync_pending);
19171            assert!(runner.state.events.iter().any(|event|
19172                event.node == "land" && event.message == format!("watching {url}")));
19173            assert_eq!(
19174                runner.state.merge.as_ref().unwrap().detail,
19175                "the fixer produced no commit"
19176            );
19177        }
19178    }
19179
19180    #[tokio::test]
19181    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
19182        crate::run::pin_test_home();
19183        let tmp = tempfile::tempdir().expect("tempdir");
19184        let repo = tmp.path().join("repo");
19185        std::fs::create_dir_all(&repo).unwrap();
19186        init_repo(&repo);
19187
19188        let mut config = Config::default();
19189        config.merge.mode = MergeMode::Pr;
19190        config.graph.land = true;
19191        config.graph.land_approval = false;
19192
19193        let mut state = RunState::new(
19194            repo.clone(),
19195            "main".to_owned(),
19196            "deadbeef".to_owned(),
19197            "task".to_owned(),
19198            config,
19199        );
19200        state.candidates = vec![Candidate {
19201            index: 0,
19202            label: 'A',
19203            agent: "alpha".to_owned(),
19204            branch: "does-not-exist".to_owned(),
19205            worktree: repo.clone(),
19206            summary: String::new(),
19207            stat: String::new(),
19208            files: 0,
19209            commits: 0,
19210            empty: false,
19211            failed: None,
19212            verified_noop: None,
19213            duration_ms: 0,
19214            folded: false,
19215        }];
19216        state.tally = Some(Tally {
19217            first_choice: BTreeMap::from([('A', 1)]),
19218            borda: BTreeMap::new(),
19219            winner: 'A',
19220            rankings: 1,
19221            unanimous_initial: true,
19222            deliberated: false,
19223            changed_votes: 0,
19224            unanimous_final: true,
19225            tie_break: None,
19226            judges: 0,
19227            present: 0,
19228            quorum: 0,
19229            met_quorum: true,
19230            uncontested: Some("only candidate A produced a change".to_owned()),
19231        });
19232        state.reviews = vec![ReviewRound {
19233            round: 1,
19234            head: "deadbeef".to_owned(),
19235            verified_head: None,
19236            verified_at: None,
19237            reviews: Vec::new(),
19238            e2e: Vec::new(),
19239            fix: None,
19240            blocking: 0,
19241            answered: 0,
19242            expected: 0,
19243            clean: true,
19244            verify_retried: false,
19245            e2e_deferred: false,
19246            e2e_defer_reason: None,
19247            progressed: false,
19248            vote_split: false,
19249            reconsideration: Vec::new(),
19250            verdict: None,
19251        }];
19252        state.gate = vec![CommandOutcome {
19253            command: "test".to_owned(),
19254            code: Some(0),
19255            output_tail: String::new(),
19256            duration_ms: 0,
19257            resource_blocked: false,
19258        }];
19259        state.gate_ran = true;
19260        // A first pass through `merge` already pushed and opened this pull
19261        // request; `status` is `Landing` because a previous call into `land`
19262        // parked or was interrupted before it reached a terminal outcome.
19263        state.status = RunStatus::Landing;
19264        state.merge = Some(MergeOutcome {
19265            mode: MergeMode::Pr,
19266            ok: true,
19267            detail: "https://example.invalid/x/y/pull/1".to_owned(),
19268            empty: false,
19269        });
19270
19271        // The Landing-resume shortcut calls `run_land` directly rather than
19272        // through `merge`, which is exactly the call site that used to skip
19273        // `settle_questions` - see the fixture below.
19274        ask_test_home();
19275        // A real resume loads a persisted run; driver ownership is checked
19276        // against that record before the Landing shortcut is entered.
19277        state.save().expect("save landing run");
19278        let store = ask::Questions::open();
19279        let q = ask_open_question(&store, &state.id);
19280
19281        let mut runner = Runner {
19282            state,
19283            roles: ResolvedRoles {
19284                implementers: Vec::new(),
19285                judges: Vec::new(),
19286                reviewers: Vec::new(),
19287                fixer: None,
19288                conductor: conductor(),
19289                implementer_roster: Vec::new(),
19290                judge_roster: Vec::new(),
19291                reviewer_roster: Vec::new(),
19292            },
19293            sem: Arc::new(Semaphore::new(1)),
19294            pause: Pause::new(),
19295            interrupt: Pause::new(),
19296        };
19297
19298        // `execute`, not `merge` directly: the Landing-resume shortcut lives
19299        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
19300        // exactly because `review_loop` would otherwise clobber the marker
19301        // first.
19302        runner.execute().await.expect("execute");
19303
19304        assert_eq!(
19305            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19306            Some("https://example.invalid/x/y/pull/1"),
19307            "reentry must not push again or open a second pull request over the \
19308             one `land` is already watching"
19309        );
19310        assert_ne!(
19311            runner.state.status,
19312            RunStatus::Landing,
19313            "land could not actually reach the fake pull request, so it must \
19314             have given up rather than left the run silently parked forever"
19315        );
19316        // `land` could not reach the fake pull request, so it gave up into
19317        // `Blocked` - still resumable, so the question must not have been
19318        // swept just because this branch now also calls `settle_questions`.
19319        assert_eq!(runner.state.status, RunStatus::Blocked);
19320        assert!(
19321            store.get(&q.id).unwrap().status.open(),
19322            "Blocked is still alive; settle_questions must have been a no-op here"
19323        );
19324    }
19325
19326    fn state_with_round(round: ReviewRound) -> RunState {
19327        let mut s = RunState::new(
19328            PathBuf::from("/repo"),
19329            "main".to_owned(),
19330            "abc1234".to_owned(),
19331            "add retries".to_owned(),
19332            Config::default(),
19333        );
19334        s.reviews = vec![round];
19335        s
19336    }
19337
19338    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
19339        crate::verdict::Finding {
19340            id: id.to_owned(),
19341            severity,
19342            file: None,
19343            line: None,
19344            title: title.to_owned(),
19345            detail: String::new(),
19346        }
19347    }
19348
19349    #[test]
19350    fn pr_body_names_open_findings_and_declined_ones() {
19351        let round = ReviewRound {
19352            round: 2,
19353            head: "deadbee".to_owned(),
19354            verified_head: None,
19355            verified_at: None,
19356            reviews: vec![ReviewRecord {
19357                attempts: 0,
19358                reviewer: 1,
19359                agent: "alpha".to_owned(),
19360                summary: String::new(),
19361                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
19362                vote: None,
19363                failed: None,
19364                duration_ms: 0,
19365            }],
19366            e2e: vec![CommandOutcome {
19367                command: "cargo test".to_owned(),
19368                code: Some(0),
19369                output_tail: String::new(),
19370                duration_ms: 0,
19371                resource_blocked: false,
19372            }],
19373            verify_retried: false,
19374            e2e_deferred: false,
19375            e2e_defer_reason: None,
19376            fix: Some(FixRecord {
19377                agent: "alpha".to_owned(),
19378                addressed: Vec::new(),
19379                rejected: vec![crate::verdict::Rejection {
19380                    id: "R1-1-1".to_owned(),
19381                    why: "not reachable from any caller".to_owned(),
19382                }],
19383                notes: String::new(),
19384                committed: true,
19385                failed: None,
19386                duration_ms: 0,
19387                continuation: None,
19388            }),
19389            blocking: 0,
19390            answered: 1,
19391            expected: 1,
19392            clean: false,
19393            progressed: true,
19394            vote_split: false,
19395            reconsideration: Vec::new(),
19396            verdict: None,
19397        };
19398        let state = state_with_round(round);
19399        let body = pr_message(&state, 'A').body;
19400
19401        assert!(body.contains("add retries"), "the task must still be there");
19402        assert!(body.contains("R2-1-1"), "{body}");
19403        assert!(!body.contains("unused import"), "{body}");
19404        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
19405        assert!(
19406            !body.contains("not reachable from any caller"),
19407            "the reason it was declined: {body}"
19408        );
19409    }
19410
19411    #[test]
19412    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
19413        let round = ReviewRound {
19414            round: 1,
19415            head: "deadbee".to_owned(),
19416            verified_head: None,
19417            verified_at: None,
19418            reviews: vec![ReviewRecord {
19419                attempts: 0,
19420                reviewer: 1,
19421                agent: "alpha".to_owned(),
19422                summary: String::new(),
19423                findings: Vec::new(),
19424                vote: None,
19425                failed: None,
19426                duration_ms: 0,
19427            }],
19428            e2e: Vec::new(),
19429            verify_retried: false,
19430            e2e_deferred: false,
19431            e2e_defer_reason: None,
19432            fix: None,
19433            blocking: 0,
19434            answered: 1,
19435            expected: 1,
19436            clean: true,
19437            progressed: false,
19438            vote_split: false,
19439            reconsideration: Vec::new(),
19440            verdict: None,
19441        };
19442        let state = state_with_round(round);
19443        let body = pr_message(&state, 'A').body;
19444        assert!(!body.contains("Open review findings"), "{body}");
19445        assert!(!body.contains("Declined"), "{body}");
19446    }
19447
19448    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
19449        let mut state = RunState::new(
19450            PathBuf::from("/repo"),
19451            "main".to_owned(),
19452            "abc1234".to_owned(),
19453            instruction.to_owned(),
19454            Config::default(),
19455        );
19456        state.candidates.push(Candidate {
19457            index: 0,
19458            label: 'A',
19459            agent: "alpha".to_owned(),
19460            branch: "magi/x/A".to_owned(),
19461            worktree: PathBuf::from("/wt"),
19462            summary: summary.to_owned(),
19463            stat: String::new(),
19464            files: 1,
19465            commits: 1,
19466            empty: false,
19467            failed: None,
19468            verified_noop: None,
19469            folded: false,
19470            duration_ms: 0,
19471        });
19472        state
19473    }
19474
19475    fn review_state(subjects: &[&str]) -> RunState {
19476        let mut state = state_with_summary(
19477            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
19478            "",
19479        );
19480        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19481        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
19482        state
19483    }
19484
19485    /// A branch rebased onto a main that moved past the recorded
19486    /// `base_commit` is titled from its own first commit, never main's.
19487    #[tokio::test]
19488    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
19489        ask_test_home();
19490        let tmp = tempfile::tempdir().unwrap();
19491        let repo = tmp.path().join("repo");
19492        std::fs::create_dir_all(&repo).unwrap();
19493        init_repo(&repo);
19494        let origin = tmp.path().join("origin.git");
19495        let g = |dir: &Path, args: &[&str]| {
19496            let out = std::process::Command::new("git")
19497                .args(args)
19498                .current_dir(dir)
19499                .quiet()
19500                .output()
19501                .expect("spawn git");
19502            assert!(
19503                out.status.success(),
19504                "git {args:?}: {}",
19505                String::from_utf8_lossy(&out.stderr)
19506            );
19507        };
19508        g(
19509            tmp.path(),
19510            &[
19511                "clone",
19512                "--bare",
19513                "-q",
19514                repo.to_str().unwrap(),
19515                origin.to_str().unwrap(),
19516            ],
19517        );
19518        g(
19519            &repo,
19520            &["remote", "add", "origin", origin.to_str().unwrap()],
19521        );
19522        let c1 = git::rev_parse(&repo, "main").await.unwrap();
19523
19524        // Main moves on; the branch is built on top of the new main.
19525        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
19526        g(&repo, &["add", "-A"]);
19527        g(
19528            &repo,
19529            &["commit", "-q", "-m", "chore(deps): update a crate"],
19530        );
19531        g(&repo, &["push", "-q", "origin", "main"]);
19532        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
19533        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19534        g(&repo, &["add", "-A"]);
19535        g(
19536            &repo,
19537            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
19538        );
19539        g(&repo, &["checkout", "-q", "main"]);
19540
19541        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
19542        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
19543        // Without a readable tracking ref the recorded base's merge base is used.
19544        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
19545        assert_eq!(fallback, c1);
19546
19547        let mut state = review_state(&[
19548            "chore(deps): update a crate",
19549            "fix(daemon): apply a chosen action",
19550        ]);
19551        state.repo = repo.clone();
19552        state.base_branch = "main".to_owned();
19553        state.base_commit = c1;
19554        refresh_reviewed_commits(&mut state, "feat/own").await;
19555        assert_eq!(
19556            state.reviewed_commits,
19557            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
19558        );
19559        assert_eq!(
19560            review_title(&state).as_deref(),
19561            Some("fix(daemon): apply a chosen action")
19562        );
19563        assert_eq!(
19564            leaked_subjects(&state, "feat/own").await,
19565            Some(vec!["chore(deps): update a crate".to_owned()])
19566        );
19567        // A stale tracking ref is still used when the fetch fails, but the
19568        // leak list is withheld.
19569        state.config.merge.remote = "nowhere".to_owned();
19570        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
19571    }
19572
19573    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
19574    /// with `origin/main` one commit behind it, and a summarizing agent that
19575    /// prints whatever `reply` holds.
19576    struct SummaryFixture {
19577        _tmp: tempfile::TempDir,
19578        repo: PathBuf,
19579        reply: PathBuf,
19580        runner: Runner,
19581    }
19582
19583    fn summary_fixture() -> SummaryFixture {
19584        ask_test_home();
19585        let tmp = tempfile::tempdir().unwrap();
19586        let repo = tmp.path().join("repo");
19587        std::fs::create_dir_all(&repo).unwrap();
19588        init_repo(&repo);
19589        let origin = tmp.path().join("origin.git");
19590        let g = |args: &[&str]| {
19591            let out = std::process::Command::new("git")
19592                .args(args)
19593                .current_dir(&repo)
19594                .quiet()
19595                .output()
19596                .expect("spawn git");
19597            assert!(
19598                out.status.success(),
19599                "git {args:?}: {}",
19600                String::from_utf8_lossy(&out.stderr)
19601            );
19602        };
19603        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
19604        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
19605        g(&["push", "-q", "origin", "main"]);
19606        g(&["checkout", "-q", "-b", "feat/own"]);
19607        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19608        g(&["add", "-A"]);
19609        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
19610        let reply = tmp.path().join("reply.txt");
19611        std::fs::write(
19612            &reply,
19613            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
19614        )
19615        .unwrap();
19616        let mut runner = runner_at(RunStatus::Gating);
19617        let mut state = review_state(&["feat(own): add the own file"]);
19618        state.repo = repo.clone();
19619        state.base_branch = "main".to_owned();
19620        state.candidates[0].branch = "feat/own".to_owned();
19621        state.candidates[0].worktree = repo.clone();
19622        let mut author = spec("alpha");
19623        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
19624        author
19625            .env
19626            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
19627        state.config.agents = vec![author];
19628        runner.state = state;
19629        SummaryFixture {
19630            _tmp: tmp,
19631            repo,
19632            reply,
19633            runner,
19634        }
19635    }
19636
19637    async fn posted(f: &mut SummaryFixture) -> PrMessage {
19638        let winner = f.runner.state.candidates[0].clone();
19639        f.runner
19640            .guarded_pr_message(&winner, None, true)
19641            .await
19642            .unwrap()
19643            .expect("an English description never parks")
19644    }
19645
19646    fn clean_round_with_minor_findings() -> ReviewRound {
19647        let mut round = passing_round(2, None);
19648        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
19649        round.reviews = vec![ReviewRecord {
19650            attempts: 0,
19651            reviewer: 1,
19652            agent: "alpha".into(),
19653            summary: "内部レビュー".into(),
19654            findings: vec![
19655                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
19656                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
19657            ],
19658            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
19659            failed: None,
19660            duration_ms: 0,
19661        }];
19662        round
19663    }
19664
19665    #[tokio::test]
19666    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
19667        for review_only in [false, true] {
19668            let mut f = summary_fixture();
19669            if !review_only {
19670                f.runner.state.candidates[0].agent = "alpha".into();
19671            }
19672            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
19673            assert!(f.runner.state.reviews[0].clean);
19674            assert!(f.runner.state.reviews[0].fix.is_none());
19675            assert!(f.runner.state.open_findings().is_empty());
19676            assert_eq!(
19677                crate::pr_outcome::required_finding_ids(&f.runner.state),
19678                ["R2-1-1", "R2-1-2"]
19679            );
19680            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19681            let evidence = crate::pr_outcome::summary_evidence(
19682                &f.runner.state,
19683                crate::pr_outcome::FinalHead::At(&head),
19684            );
19685            assert!(
19686                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
19687            );
19688            // The ordinary reply omits both IDs: no cached description is adopted.
19689            let omitted = posted(&mut f).await;
19690            assert!(omitted.body.contains("No description"));
19691            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
19692            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
19693            assert!(f.runner.state.change_summary.is_none());
19694            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
19695            let described = posted(&mut f).await;
19696            assert!(described.body.contains("improve documentation for callers"));
19697            assert!(described.body.contains("remaining edge case"));
19698            assert!(described.body.contains("## Open review findings"));
19699            assert!(!described.body.contains("未修正"));
19700        }
19701    }
19702
19703    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
19704        let mut f = summary_fixture();
19705        if !review_only {
19706            f.runner.state.candidates[0].agent = "alpha".into();
19707        }
19708        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19709        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
19710        fixed
19711            .fix
19712            .as_mut()
19713            .unwrap()
19714            .rejected
19715            .push(crate::verdict::Rejection {
19716                id: "R1-1-1".into(),
19717                why: "呼び出されないコードのため辞退".into(),
19718            });
19719        let mut clean = clean_round_with_minor_findings();
19720        clean.verified_head = Some(head.clone());
19721        f.runner.state.reviews = vec![fixed, clean];
19722        ok_gate(&mut f.runner.state, Some(&head));
19723        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
19724        f.runner.state.config.agents.push(scripted("beta", reply));
19725        f.runner.state.config.roles.language_judge =
19726            Some(crate::config::AgentChoice::One("beta".into()));
19727        f
19728    }
19729
19730    #[tokio::test]
19731    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
19732        for review_only in [false, true] {
19733            let mut f = rewrite_fixture_with_findings(review_only,
19734                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
19735            let m = posted(&mut f).await;
19736            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19737            let expected =
19738                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
19739            assert!(m.body.contains(&expected));
19740            assert!(m.body.contains("## Review rounds"));
19741            assert!(
19742                m.body
19743                    .contains("Gate (`verify.gate`): passed on the final commit")
19744            );
19745            assert!(
19746                m.body
19747                    .contains("Tests (`verify.e2e`): passed on the final commit")
19748            );
19749            let c = f.runner.state.change_summary.as_ref().unwrap();
19750            assert!(
19751                m.body
19752                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
19753            );
19754            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19755            assert!(crate::pr_outcome::covers_required_findings(
19756                &f.runner.state,
19757                &m.body
19758            ));
19759            assert!(m.body.contains("documentation remains incomplete"));
19760            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
19761            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
19762            assert!(f.runner.state.github_text.is_none());
19763        }
19764    }
19765
19766    #[tokio::test]
19767    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
19768        for review_only in [false, true] {
19769            for reply in [
19770                "TITLE: fix: public description\nA short English description.",
19771                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
19772            ] {
19773                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
19774                let winner = f.runner.state.candidates[0].clone();
19775                let posted = f
19776                    .runner
19777                    .guarded_pr_message(&winner, None, true)
19778                    .await
19779                    .unwrap();
19780                assert!(posted.is_none());
19781                assert!(f.runner.state.parked);
19782                let withheld = f.runner.state.github_text.as_ref().unwrap();
19783                assert!(
19784                    withheld.rewrite.is_none(),
19785                    "an incomplete rewrite must not be selectable"
19786                );
19787                let q = ask::Questions::open()
19788                    .get(withheld.question.as_ref().unwrap())
19789                    .unwrap();
19790                assert!(
19791                    !q.choices
19792                        .contains(&crate::github_text::USE_REWRITE.to_owned())
19793                );
19794                assert!(
19795                    q.detail.contains("R1-1-1")
19796                        && q.detail.contains("R2-1-1")
19797                        && q.detail.contains("R2-1-2")
19798                );
19799            }
19800        }
19801    }
19802
19803    #[tokio::test]
19804    async fn japanese_reports_stay_local_in_both_public_run_modes() {
19805        for review_only in [false, true] {
19806            let mut f = summary_fixture();
19807            if !review_only {
19808                f.runner.state.candidates[0].agent = "alpha".into();
19809            }
19810            f.runner.state.instruction = "内部タスクの説明".into();
19811            f.runner.state.candidates[0].summary = "内部の候補報告".into();
19812            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19813            f.runner.state.reviews = vec![
19814                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
19815                round_with_notes(2, Some("二回目の内部修正報告です。")),
19816                passing_round(3, Some(&head)),
19817            ];
19818            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
19819            ok_gate(&mut f.runner.state, Some(&head));
19820            let captured = f.repo.join("public-prompt.txt");
19821            let author = &mut f.runner.state.config.agents[0];
19822            author.command = vec![
19823                "sh".into(),
19824                "-c".into(),
19825                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19826            ];
19827            author.env.insert(
19828                "PROMPT_FILE".into(),
19829                captured.to_string_lossy().into_owned(),
19830            );
19831            let m = posted(&mut f).await;
19832            assert!(m.body.contains("Adds `own.txt`"));
19833            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
19834            assert!(m.body.contains("R1-1-1"));
19835            assert!(
19836                m.body
19837                    .contains("Gate (`verify.gate`): passed on the final commit")
19838            );
19839            assert!(
19840                m.body
19841                    .contains("Tests (`verify.e2e`): passed on the final commit")
19842            );
19843            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19844            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
19845            let prompt = std::fs::read_to_string(captured).unwrap();
19846            assert!(prompt.contains(&format!(
19847                "git diff {} {head}",
19848                f.runner.state.change_summary.as_ref().unwrap().base
19849            )));
19850            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
19851            assert_eq!(
19852                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
19853                "二回目の内部修正報告です。"
19854            );
19855        }
19856    }
19857
19858    #[tokio::test]
19859    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
19860        for review_only in [false, true] {
19861            let mut f = summary_fixture();
19862            if !review_only {
19863                f.runner.state.candidates[0].agent = "alpha".into();
19864            }
19865            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
19866            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
19867            std::fs::write(&f.reply, "no usable description").unwrap();
19868            let failed = posted(&mut f).await;
19869            assert!(
19870                failed
19871                    .body
19872                    .contains("No description of what this change does could be generated")
19873            );
19874            assert!(failed.body.is_ascii());
19875            assert!(f.runner.state.github_text.is_none());
19876            assert!(f.runner.state.change_summary.is_none());
19877            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19878            let evidence = crate::pr_outcome::summary_evidence(
19879                &f.runner.state,
19880                crate::pr_outcome::FinalHead::At(&head),
19881            );
19882            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
19883            assert!(
19884                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
19885                    .exists()
19886            );
19887            f.runner.state =
19888                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
19889            std::fs::write(
19890                &f.reply,
19891                "TITLE: feat(own): add the own file\nAdds the committed own file.",
19892            )
19893            .unwrap();
19894            let retried = posted(&mut f).await;
19895            assert!(retried.body.contains("Adds the committed own file."));
19896            assert_eq!(
19897                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
19898                "日本語の修正報告"
19899            );
19900        }
19901    }
19902
19903    #[tokio::test]
19904    async fn same_head_verification_and_findings_invalidate_public_prose() {
19905        let mut f = summary_fixture();
19906        posted(&mut f).await;
19907        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19908        let original = f
19909            .runner
19910            .state
19911            .change_summary
19912            .as_ref()
19913            .unwrap()
19914            .evidence
19915            .clone();
19916        ok_gate(&mut f.runner.state, Some(&head));
19917        std::fs::write(
19918            &f.reply,
19919            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
19920        )
19921        .unwrap();
19922        let verified = posted(&mut f).await;
19923        assert!(verified.body.contains("now verified"));
19924        assert_ne!(
19925            f.runner.state.change_summary.as_ref().unwrap().evidence,
19926            original
19927        );
19928        let mut round = round_with_notes(1, Some("内部報告"));
19929        round.reviews.push(ReviewRecord {
19930            attempts: 0,
19931            reviewer: 1,
19932            agent: "alpha".into(),
19933            summary: "内部レビュー".into(),
19934            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
19935            vote: None,
19936            failed: None,
19937            duration_ms: 0,
19938        });
19939        round
19940            .fix
19941            .as_mut()
19942            .unwrap()
19943            .rejected
19944            .push(crate::verdict::Rejection {
19945                id: "R1-1-1".into(),
19946                why: "呼び出し元から到達できない".into(),
19947            });
19948        f.runner.state.reviews = vec![round];
19949        // A description omitting the required ID is unusable, even with a title.
19950        let omitted = posted(&mut f).await;
19951        assert!(omitted.body.contains("No description"));
19952        assert!(omitted.body.contains("R1-1-1"));
19953        assert!(!omitted.body.contains("到達"));
19954        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
19955        let described = posted(&mut f).await;
19956        assert!(described.body.contains("unreachable from callers"));
19957        assert!(described.body.contains("R1-1-1"));
19958        assert!(!described.body.contains("到達"));
19959        let evidence = f
19960            .runner
19961            .state
19962            .change_summary
19963            .as_ref()
19964            .unwrap()
19965            .evidence
19966            .clone();
19967        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
19968        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
19969        assert!(posted(&mut f).await.body.contains("rationale updated"));
19970        assert_ne!(
19971            f.runner.state.change_summary.as_ref().unwrap().evidence,
19972            evidence
19973        );
19974        // Old serialized summaries have no evidence fingerprint and must regenerate.
19975        f.runner
19976            .state
19977            .change_summary
19978            .as_mut()
19979            .unwrap()
19980            .evidence
19981            .clear();
19982        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
19983        assert!(posted(&mut f).await.body.contains("current specification"));
19984    }
19985
19986    #[tokio::test]
19987    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
19988        let mut f = summary_fixture();
19989        let m = posted(&mut f).await;
19990        assert_eq!(m.title, "feat(own): add the own file");
19991        assert!(
19992            m.body
19993                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
19994            "{}",
19995            m.body
19996        );
19997        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
19998        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
19999        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
20000        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20001        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
20002        let kept = f.runner.state.change_summary.as_ref().unwrap();
20003        assert_eq!(
20004            (kept.head.as_str(), kept.base.as_str()),
20005            (head.as_str(), base.as_str())
20006        );
20007        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
20008    }
20009
20010    #[tokio::test]
20011    async fn review_only_summary_receives_the_configured_github_language() {
20012        let mut f = summary_fixture();
20013        f.runner.state.config.graph.github_language = "ja".into();
20014        let captured = f.repo.join("summary-prompt.txt");
20015        let author = &mut f.runner.state.config.agents[0];
20016        author.command = vec![
20017            "sh".into(),
20018            "-c".into(),
20019            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
20020        ];
20021        author.env.insert(
20022            "PROMPT_FILE".into(),
20023            captured.to_string_lossy().into_owned(),
20024        );
20025        std::fs::write(
20026            &f.reply,
20027            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
20028        )
20029        .unwrap();
20030        let m = posted(&mut f).await;
20031        let prompt = std::fs::read_to_string(captured).unwrap();
20032        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
20033        assert!(!prompt.contains("in English"));
20034        assert_eq!(m.title, "feat(own): ファイルを追加");
20035        assert!(m.body.contains("own.txt を追加します。"));
20036    }
20037
20038    #[tokio::test]
20039    async fn review_fixes_come_after_the_whole_change_summary() {
20040        let mut f = summary_fixture();
20041        f.runner.state.reviews = vec![
20042            round_with_notes(1, Some("handled the PATH case")),
20043            round_with_notes(2, None),
20044        ];
20045        let m = posted(&mut f).await;
20046        let summary = m.body.find("## Summary").expect("summary");
20047        let fixes = m.body.find("## Review rounds").expect("review rounds");
20048        assert!(summary < fixes, "{}", m.body);
20049        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
20050        assert!(m.body.contains("## Final verification"), "{}", m.body);
20051    }
20052
20053    #[tokio::test]
20054    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
20055        let mut f = summary_fixture();
20056        let first = posted(&mut f).await;
20057        let first_head = f.runner.state.change_summary.clone().unwrap().head;
20058        // The same head reuses the record: a different reply is not read.
20059        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
20060        let again = posted(&mut f).await;
20061        assert_eq!((again.title, again.body), (first.title, first.body));
20062
20063        let g = |args: &[&str]| {
20064            let out = std::process::Command::new("git")
20065                .args(args)
20066                .current_dir(&f.repo)
20067                .quiet()
20068                .output()
20069                .expect("spawn git");
20070            assert!(out.status.success(), "git {args:?}");
20071        };
20072        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
20073        g(&["add", "-A"]);
20074        g(&["commit", "-q", "-m", "fix(own): handle more"]);
20075        let second = posted(&mut f).await;
20076        let kept = f.runner.state.change_summary.clone().unwrap();
20077        assert_ne!(kept.head, first_head);
20078        assert!(second.body.contains("Not read yet."), "{}", second.body);
20079        assert!(!second.body.contains("so the first summary is read"));
20080        assert_eq!(second.title, "feat(own): other");
20081    }
20082
20083    #[tokio::test]
20084    async fn an_unavailable_summary_is_stated_and_never_invented() {
20085        let mut f = summary_fixture();
20086        std::fs::write(&f.reply, "no title line here").unwrap();
20087        let m = posted(&mut f).await;
20088        assert!(
20089            m.body
20090                .contains("No description of what this change does could be generated"),
20091            "{}",
20092            m.body
20093        );
20094        assert!(m.body.contains("internal reports remain in local records"));
20095        assert!(f.runner.state.change_summary.is_none());
20096        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
20097        // A preview asks nobody and claims nothing.
20098        let winner = f.runner.state.candidates[0].clone();
20099        let preview = f
20100            .runner
20101            .guarded_pr_message(&winner, None, false)
20102            .await
20103            .unwrap()
20104            .unwrap();
20105        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
20106    }
20107
20108    #[test]
20109    fn pr_message_review_single_commit_uses_its_subject() {
20110        let state = review_state(&["feat(nats): per-role user"]);
20111        let m = pr_message(&state, 'A');
20112        assert_eq!(m.title, "feat(nats): per-role user");
20113        assert!(!m.body.contains("Review the work already"), "{}", m.body);
20114        assert!(m.body.contains("## Commits under review"), "{}", m.body);
20115    }
20116
20117    #[test]
20118    fn pr_message_review_multi_commit_takes_the_oldest() {
20119        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
20120        let m = pr_message(&state, 'A');
20121        assert_eq!(m.title, "feat: the change");
20122        for s in ["feat: the change", "fix: typo", "fix: again"] {
20123            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
20124        }
20125    }
20126
20127    #[test]
20128    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
20129        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
20130            let mut state = review_state(&[first, "fix: later fixup"]);
20131            state.candidates[0].branch = "機能/ログイン".to_owned();
20132            let m = pr_message(&state, 'A');
20133            assert!(
20134                m.title.starts_with("chore: land candidate A of run"),
20135                "{}",
20136                m.title
20137            );
20138        }
20139    }
20140
20141    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
20142        BranchFacts {
20143            commits: commits
20144                .iter()
20145                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
20146                .collect(),
20147            stat: stat.to_owned(),
20148        }
20149    }
20150
20151    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
20152        let mut r = review_round(true, 0, 1, 1, true, true);
20153        r.round = round;
20154        r.fix = notes.map(|n| FixRecord {
20155            agent: "fixer".to_owned(),
20156            addressed: Vec::new(),
20157            rejected: Vec::new(),
20158            notes: n.to_owned(),
20159            committed: true,
20160            failed: None,
20161            duration_ms: 0,
20162            continuation: None,
20163        });
20164        r
20165    }
20166
20167    #[test]
20168    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
20169        let state = review_state(&["ignored"]);
20170        let f = facts(
20171            &[
20172                (
20173                    "fix(login): resolve PATH on macOS",
20174                    "Login shells skip rc files.",
20175                ),
20176                ("fix: address review", ""),
20177            ],
20178            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
20179        );
20180        let m = pr_message_with(&state, 'A', Some(&f));
20181        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
20182        assert!(
20183            m.body
20184                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
20185            "{}",
20186            m.body
20187        );
20188        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
20189        assert!(m.body.contains("## Diff stat"), "{}", m.body);
20190        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
20191        for banned in [
20192            "Review the work already",
20193            "no task statement",
20194            "Original task",
20195        ] {
20196            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
20197        }
20198        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
20199    }
20200
20201    #[test]
20202    fn pr_message_review_truncates_a_huge_first_commit_body() {
20203        let state = review_state(&["ignored"]);
20204        let f = facts(
20205            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
20206            "s",
20207        );
20208        let m = pr_message_with(&state, 'A', Some(&f));
20209        assert!(m.body.len() < 30_000, "{}", m.body.len());
20210        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
20211        assert!(
20212            m.body.contains("- ... 1 more commit(s)"),
20213            "{}",
20214            m.body.len()
20215        );
20216        assert!(m.body.ends_with("magi:candidate-a\n"));
20217    }
20218
20219    #[test]
20220    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
20221        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
20222        assert_eq!(m.title, "feat: x");
20223        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
20224        assert!(!m.body.contains("Diff stat"), "{}", m.body);
20225        assert!(!m.body.contains("no task statement"), "{}", m.body);
20226    }
20227
20228    #[test]
20229    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
20230        let mut state = review_state(&["日本語の件名"]);
20231        state.candidates[0].branch = "fix/macos-login-path".to_owned();
20232        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
20233        state.candidates[0].branch = "機能/ログイン".to_owned();
20234        assert!(
20235            pr_message(&state, 'A')
20236                .title
20237                .starts_with("chore: land candidate A")
20238        );
20239    }
20240
20241    #[test]
20242    fn pr_message_review_fixes_survive_a_clean_final_round() {
20243        let mut state = review_state(&["feat: x"]);
20244        state.reviews = vec![
20245            round_with_notes(1, Some("handled the PATH case")),
20246            round_with_notes(2, None),
20247        ];
20248        let body = pr_message(&state, 'A').body;
20249        assert!(
20250            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
20251            "{body}"
20252        );
20253        assert!(body.contains("### Round 1"), "{body}");
20254
20255        state.reviews = vec![
20256            round_with_notes(1, Some("first fix")),
20257            round_with_notes(2, Some("")),
20258            round_with_notes(3, Some("second fix")),
20259            round_with_notes(4, None),
20260        ];
20261        let body = pr_message(&state, 'A').body;
20262        assert!(
20263            body.contains("### Round 1") && !body.contains("first fix"),
20264            "{body}"
20265        );
20266        assert!(
20267            body.contains("### Round 3") && !body.contains("second fix"),
20268            "{body}"
20269        );
20270        assert!(body.contains("### Round 2"), "{body}");
20271    }
20272
20273    const FINAL: &str = "abcdef1234567890";
20274
20275    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
20276        let mut r = round_with_notes(round, None);
20277        r.verified_head = verified.map(str::to_owned);
20278        r
20279    }
20280
20281    fn ok_gate(state: &mut RunState, head: Option<&str>) {
20282        state.gate = vec![CommandOutcome {
20283            command: "gate".to_owned(),
20284            code: Some(0),
20285            output_tail: String::new(),
20286            duration_ms: 0,
20287            resource_blocked: false,
20288        }];
20289        state.gate_ran = true;
20290        state.gate_head = head.map(str::to_owned);
20291    }
20292
20293    fn body_at(state: &RunState) -> String {
20294        pr_message_raw(
20295            state,
20296            'A',
20297            None,
20298            crate::pr_outcome::FinalHead::At(FINAL),
20299            ChangeNote::NotAsked,
20300        )
20301        .body
20302    }
20303
20304    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
20305
20306    #[test]
20307    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
20308        let mut state = review_state(&["feat: x"]);
20309        state.reviews = vec![
20310            round_with_notes(1, Some(SANDBOXED)),
20311            passing_round(2, Some(FINAL)),
20312        ];
20313        ok_gate(&mut state, Some(FINAL));
20314        let body = body_at(&state);
20315        assert!(!body.contains("Handled the PATH case."), "{body}");
20316        assert!(!body.contains("could not commit"), "{body}");
20317        assert!(!body.contains("could not run"), "{body}");
20318        assert!(
20319            body.contains("The fix is committed on the branch"),
20320            "{body}"
20321        );
20322        assert!(!body.contains("sandbox"), "{body}");
20323        assert!(
20324            body.contains("Tests (`verify.e2e`): passed on the final commit"),
20325            "{body}"
20326        );
20327        assert!(
20328            body.contains("Gate (`verify.gate`): passed on the final commit"),
20329            "{body}"
20330        );
20331        // The raw report is untouched in the record.
20332        assert!(
20333            state.reviews[0]
20334                .fix
20335                .as_ref()
20336                .unwrap()
20337                .notes
20338                .contains("could not commit")
20339        );
20340    }
20341
20342    #[test]
20343    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
20344        let mut state = review_state(&["feat: x"]);
20345        let mut r1 = round_with_notes(1, Some(SANDBOXED));
20346        r1.fix.as_mut().unwrap().committed = false;
20347        let mut r2 = passing_round(2, Some(FINAL));
20348        r2.e2e[0].code = Some(1);
20349        state.reviews = vec![r1, r2];
20350        let body = body_at(&state);
20351        assert!(!body.contains("could not commit"), "{body}");
20352        assert!(!body.contains("could not run"), "{body}");
20353        assert!(body.contains("This round produced no commit"), "{body}");
20354        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
20355        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
20356        assert!(!body.contains("passed"), "{body}");
20357    }
20358
20359    #[test]
20360    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
20361        let mut state = review_state(&["feat: x"]);
20362        state.reviews = vec![
20363            round_with_notes(1, Some(SANDBOXED)),
20364            passing_round(2, Some("0123456789abcdef")),
20365        ];
20366        ok_gate(&mut state, Some("0123456789abcdef"));
20367        let body = body_at(&state);
20368        assert!(!body.contains("HTTP tests could not run"), "{body}");
20369        // The commit claim is still settled by the structured record.
20370        assert!(!body.contains("could not commit"), "{body}");
20371        assert!(
20372            body.contains("passed on an earlier commit (`0123456`)"),
20373            "{body}"
20374        );
20375        assert!(!body.contains("passed on the final commit"), "{body}");
20376
20377        // A gate recorded before `gate_head` existed is "not recorded".
20378        ok_gate(&mut state, None);
20379        let body = body_at(&state);
20380        assert!(
20381            body.contains("passed on a commit that was not recorded"),
20382            "{body}"
20383        );
20384    }
20385
20386    #[test]
20387    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
20388        let state = state_with_summary(
20389            "add retries to the client",
20390            "TITLE: feat: retries\n\nDid it.",
20391        );
20392        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
20393        assert_eq!(m.title, "feat: retries");
20394        assert!(
20395            m.body.contains("<summary>Original task</summary>"),
20396            "{}",
20397            m.body
20398        );
20399        assert!(!m.body.contains("Commits under review"), "{}", m.body);
20400        assert!(
20401            m.body
20402                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
20403            "{}",
20404            m.body
20405        );
20406    }
20407
20408    #[test]
20409    fn pr_message_review_bounds_a_long_english_subject() {
20410        let long = format!("feat: {}", "word ".repeat(100));
20411        let m = pr_message(&review_state(&[&long]), 'A');
20412        assert!(m.title.starts_with("feat: word"), "{}", m.title);
20413        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20414    }
20415
20416    #[test]
20417    fn pr_message_implementation_run_is_unchanged_by_review_support() {
20418        let state = state_with_summary("add retries\n\ndetails", "- did some things");
20419        let m = pr_message(&state, 'A');
20420        assert_eq!(m.title, "add retries");
20421        assert!(m.body.contains("<summary>Original task</summary>"));
20422        assert!(!m.body.contains("Commits under review"));
20423        assert_eq!(landing_subject_source(&state), state.instruction);
20424    }
20425
20426    #[test]
20427    fn review_run_squash_subject_is_the_change_not_the_prompt() {
20428        let state = review_state(&["feat: the change", "fix: typo"]);
20429        let source = landing_subject_source(&state);
20430        assert_eq!(land::merge_subject("", &source), "feat: the change");
20431        assert_eq!(
20432            land::merge_subject("magi: candidate A (uncommitted work)", &source),
20433            "feat: the change"
20434        );
20435        // An operator's rename still wins.
20436        assert_eq!(
20437            land::merge_subject("feat: renamed by hand", &source),
20438            "feat: renamed by hand"
20439        );
20440        let blank = review_state(&["日本語"]);
20441        assert!(
20442            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
20443        );
20444    }
20445
20446    #[test]
20447    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
20448        let state = review_state(&["feat: the change"]);
20449        let source = landing_subject_source(&state);
20450        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
20451        assert_eq!(
20452            land::merge_subject(landing_title(&state, old), &source),
20453            "feat: the change"
20454        );
20455        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
20456        let task = state_with_summary("add retries", "");
20457        assert_eq!(landing_title(&task, old), old);
20458    }
20459
20460    #[test]
20461    fn pr_message_describes_the_change_not_the_task() {
20462        let state = state_with_summary(
20463            "今回やってほしいこと: results projector を直す",
20464            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
20465        );
20466        let m = pr_message(&state, 'A');
20467        assert_eq!(m.title, "fix(web): batch the runs list reads");
20468        assert!(
20469            m.body.starts_with("## Summary\n\n- reads run.json once"),
20470            "{}",
20471            m.body
20472        );
20473        assert!(!m.body.contains("TITLE:"), "{}", m.body);
20474        let task_at = m.body.find("今回やってほしいこと").unwrap();
20475        let details_at = m.body.find("<details>").unwrap();
20476        assert!(
20477            details_at < task_at,
20478            "the task lives inside <details>: {}",
20479            m.body
20480        );
20481        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
20482        assert!(m.body.contains("magi:candidate-a"));
20483    }
20484
20485    #[test]
20486    fn pr_message_falls_back_to_the_task_without_a_title_line() {
20487        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
20488        let m = pr_message(&state, 'A');
20489        assert_eq!(m.title, "add retries");
20490        assert!(
20491            m.body.contains("## Summary\n\n- did some things"),
20492            "{}",
20493            m.body
20494        );
20495
20496        let none = RunState::new(
20497            PathBuf::from("/repo"),
20498            "main".to_owned(),
20499            "abc1234".to_owned(),
20500            "add retries".to_owned(),
20501            Config::default(),
20502        );
20503        let m = pr_message(&none, 'A');
20504        assert_eq!(m.title, "add retries");
20505        assert!(!m.body.contains("## Summary"), "{}", m.body);
20506    }
20507
20508    #[test]
20509    fn pr_message_refuses_the_candidate_commit_subject() {
20510        for bad in [
20511            "TITLE: magi: candidate A (uncommitted work)",
20512            "TITLE: chore: stuff (uncommitted work)",
20513            "TITLE:   ",
20514        ] {
20515            let state = state_with_summary("add retries", bad);
20516            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
20517        }
20518    }
20519
20520    #[test]
20521    fn pr_message_bounds_a_very_long_task_and_title() {
20522        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
20523        let state = state_with_summary(&long, "- nothing");
20524        let m = pr_message(&state, 'A');
20525        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20526        assert!(!m.title.contains('\n'));
20527
20528        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
20529        let m = pr_message(&state, 'A');
20530        assert!(m.title.starts_with("feat: "));
20531        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20532        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
20533    }
20534
20535    fn long_title_of(instruction: &str) -> String {
20536        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
20537    }
20538
20539    #[test]
20540    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
20541        let first = "Make the landing path keep a readable title for long tasks";
20542        let line = format!(
20543            "{first}. {}",
20544            "And then keep going with more words ".repeat(20)
20545        );
20546        let t = long_title_of(&line);
20547        assert_eq!(t, first);
20548        assert!(!t.starts_with("chore: land"));
20549    }
20550
20551    #[test]
20552    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
20553        let line = "word ".repeat(200);
20554        let t = long_title_of(&line);
20555        assert!(t.ends_with("word..."), "{t}");
20556        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
20557    }
20558
20559    #[test]
20560    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
20561        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
20562            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
20563        }
20564    }
20565
20566    #[test]
20567    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
20568        let mut state = state_with_summary("再試行を追加する", "- nothing");
20569        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
20570        state.config.graph.github_language = "ja".into();
20571        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
20572    }
20573
20574    #[test]
20575    fn pr_message_title_limit_is_exact() {
20576        let at = "a".repeat(PR_TITLE_MAX);
20577        assert_eq!(long_title_of(&at), at);
20578        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
20579        assert!(over.ends_with("..."), "{over}");
20580        assert_eq!(over.chars().count(), PR_TITLE_MAX);
20581    }
20582
20583    #[test]
20584    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
20585        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
20586        let t = long_title_of(&line);
20587        assert!(t.ends_with("..."), "{t}");
20588        assert!(t.is_ascii(), "{t}");
20589    }
20590
20591    #[test]
20592    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
20593        let line = format!(
20594            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
20595            "plus more filler words ".repeat(20)
20596        );
20597        let t = long_title_of(&line);
20598        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
20599    }
20600
20601    #[test]
20602    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
20603        let line = format!("\n\n# {}", "title words ".repeat(40));
20604        let state = state_with_summary(&line, "- nothing");
20605        let m = pr_message(&state, 'A');
20606        assert!(m.title.starts_with("title words"), "{}", m.title);
20607        assert_eq!(
20608            land::merge_subject(&m.title, &landing_subject_source(&state)),
20609            m.title
20610        );
20611        // An operator's rename wins untouched.
20612        assert_eq!(
20613            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
20614            "feat: renamed by hand"
20615        );
20616    }
20617
20618    #[tokio::test]
20619    async fn github_text_rewrite_is_bounded_and_falls_back() {
20620        crate::run::pin_test_home();
20621        for (reply, accepted) in [
20622            (
20623                "TITLE: fix: retries\nAdd retries for failed requests.",
20624                true,
20625            ),
20626            (
20627                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
20628                false,
20629            ),
20630            (
20631                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
20632                true,
20633            ),
20634            (
20635                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
20636                true,
20637            ),
20638            (
20639                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
20640                true,
20641            ),
20642            (
20643                "TITLE: fix: retries\n日本語の説明です。 token=secret",
20644                false,
20645            ),
20646            ("TITLE: fix: retries\n", false),
20647        ] {
20648            let dir = tempfile::tempdir().unwrap();
20649            let mut runner = runner_at(RunStatus::Gating);
20650            runner.state = state_with_summary(
20651                "add retries",
20652                "TITLE: fix: retries\n日本語の説明を書きます。",
20653            );
20654            runner.state.repo = dir.path().to_owned();
20655            runner.state.candidates[0].worktree = dir.path().to_owned();
20656            let mut author = spec("alpha");
20657            author.command = vec![
20658                "sh".into(),
20659                "-c".into(),
20660                "printf '%s' \"$REWRITE_REPLY\"".into(),
20661            ];
20662            author.env.insert("REWRITE_REPLY".into(), reply.into());
20663            runner.state.config.agents = vec![author];
20664            seed_public_summary(&mut runner);
20665            let winner = runner.state.candidates[0].clone();
20666            let first = runner
20667                .guarded_pr_message(&winner, None, true)
20668                .await
20669                .unwrap();
20670            let message = if accepted {
20671                first.expect("a passing rewrite never asks")
20672            } else {
20673                // Withheld: the run parks on a question and posts nothing yet.
20674                assert!(first.is_none());
20675                assert!(runner.state.parked);
20676                let store = ask::Questions::open();
20677                let asked: Vec<_> = store
20678                    .list()
20679                    .into_iter()
20680                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20681                    .collect();
20682                assert_eq!(asked.len(), 1);
20683                assert!(!asked[0].detail.contains("secret"));
20684                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
20685                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
20686                assert_eq!(
20687                    asked[0]
20688                        .choices
20689                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
20690                    rewrite.is_some()
20691                );
20692                // Silence: the question lapses and the neutral text is used,
20693                // without a second rewrite call or a second question.
20694                store
20695                    .update(&asked[0].id, |q| {
20696                        q.abandon("test");
20697                        Ok(())
20698                    })
20699                    .unwrap();
20700                let again = runner
20701                    .guarded_pr_message(&winner, None, true)
20702                    .await
20703                    .unwrap()
20704                    .expect("an abandoned question falls back");
20705                let n = store
20706                    .list()
20707                    .into_iter()
20708                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20709                    .count();
20710                assert_eq!(n, 1, "asked once per rejected text");
20711                again
20712            };
20713            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
20714            assert!(!message.body.contains("secret"));
20715            assert!(!message.body.contains("prefix"));
20716            assert!(!message.title.contains("secret"));
20717            assert_eq!(
20718                message.body.contains("Add retries for failed requests."),
20719                accepted
20720            );
20721            assert_eq!(
20722                runner.state.seats["github-text-rewrite"].turns, 1,
20723                "only one rewrite invocation"
20724            );
20725            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
20726            if accepted {
20727                assert!(runner.state.github_text.is_none());
20728                assert!(!runner.state.parked);
20729                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
20730                assert!(message.body.contains("## Final verification"));
20731                assert!(message.body.contains("Evidence: final diff"));
20732                assert!(
20733                    message
20734                        .body
20735                        .contains(&format!("magi:run/{}", runner.state.id))
20736                );
20737            }
20738            if !accepted {
20739                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20740                assert!(
20741                    message
20742                        .body
20743                        .contains(&format!("magi:run/{}", runner.state.id))
20744                );
20745            }
20746        }
20747    }
20748
20749    fn scripted(id: &str, reply: &str) -> AgentSpec {
20750        let mut a = spec(id);
20751        a.command = vec![
20752            "sh".into(),
20753            "-c".into(),
20754            "printf '%s' \"$REWRITE_REPLY\"".into(),
20755        ];
20756        a.env.insert("REWRITE_REPLY".into(), reply.into());
20757        a
20758    }
20759
20760    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
20761    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
20762
20763    /// Gate tests inject generated public output, not internal reports.
20764    fn seed_public_summary(runner: &mut Runner) {
20765        let repo = &runner.state.repo;
20766        let g = |args: &[&str]| {
20767            let out = std::process::Command::new("git")
20768                .args(args)
20769                .current_dir(repo)
20770                .quiet()
20771                .output()
20772                .unwrap();
20773            assert!(out.status.success(), "git {args:?}");
20774            String::from_utf8_lossy(&out.stdout).trim().to_owned()
20775        };
20776        g(&["init", "-q", "-b", "main"]);
20777        g(&[
20778            "-c",
20779            "user.name=Test",
20780            "-c",
20781            "user.email=test@example.invalid",
20782            "commit",
20783            "-q",
20784            "--allow-empty",
20785            "-m",
20786            "feat: fixture",
20787        ]);
20788        let head = g(&["rev-parse", "HEAD"]);
20789        g(&["update-ref", "refs/remotes/origin/main", &head]);
20790        let evidence = crate::pr_outcome::summary_evidence(
20791            &runner.state,
20792            crate::pr_outcome::FinalHead::At(&head),
20793        );
20794        let generated = &runner.state.candidates[0].summary;
20795        runner.state.change_summary = Some(crate::run::ChangeSummary {
20796            base: head.clone(),
20797            head,
20798            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
20799            title: summary_title(generated).unwrap(),
20800            body: summary_without_title(generated),
20801        });
20802    }
20803
20804    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
20805        let mut runner = runner_at(RunStatus::Gating);
20806        runner.state = state_with_summary("add retries", JA_SUMMARY);
20807        if review_only {
20808            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
20809        }
20810        runner.state.repo = dir.to_owned();
20811        runner.state.candidates[0].worktree = dir.to_owned();
20812        runner.state.config.agents = agents;
20813        seed_public_summary(&mut runner);
20814        runner
20815    }
20816
20817    fn gate_questions(runner: &Runner) -> usize {
20818        ask::Questions::open()
20819            .list()
20820            .into_iter()
20821            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20822            .count()
20823    }
20824
20825    #[tokio::test]
20826    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
20827        crate::run::pin_test_home();
20828        let dir = tempfile::tempdir().unwrap();
20829        let mut runner = rewrite_runner(
20830            dir.path(),
20831            true,
20832            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20833        );
20834        runner.state.config.roles.language_judge =
20835            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
20836        let winner = runner.state.candidates[0].clone();
20837        let msg = runner
20838            .guarded_pr_message(&winner, None, true)
20839            .await
20840            .unwrap()
20841            .expect("an English rewrite posts without asking");
20842        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
20843        assert!(msg.body.contains("<details>kept</details>"));
20844        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
20845        assert_eq!(gate_questions(&runner), 0);
20846        assert!(runner.state.github_text.is_none());
20847        assert!(!runner.state.seats.contains_key("impl-A"));
20848        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20849    }
20850
20851    #[tokio::test]
20852    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
20853        crate::run::pin_test_home();
20854        let dir = tempfile::tempdir().unwrap();
20855        let mut runner = rewrite_runner(
20856            dir.path(),
20857            false,
20858            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20859        );
20860        runner.state.config.roles.language_judge =
20861            Some(crate::config::AgentChoice::One("beta".into()));
20862        runner.state.seats.insert(
20863            "impl-A".into(),
20864            SeatState::new("impl-A", "alpha", runner.state.seed),
20865        );
20866        let winner = runner.state.candidates[0].clone();
20867        let msg = runner
20868            .guarded_pr_message(&winner, None, true)
20869            .await
20870            .unwrap()
20871            .expect("rewritten");
20872        assert!(msg.body.contains("<details>kept</details>"));
20873        assert_eq!(runner.state.seats["impl-A"].turns, 0);
20874        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20875        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20876    }
20877
20878    #[tokio::test]
20879    async fn rewrite_reuses_a_resumable_implementer_session() {
20880        crate::run::pin_test_home();
20881        let dir = tempfile::tempdir().unwrap();
20882        let mut runner = rewrite_runner(
20883            dir.path(),
20884            false,
20885            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
20886        );
20887        runner.state.config.roles.language_judge =
20888            Some(crate::config::AgentChoice::One("beta".into()));
20889        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20890        seat.turns = 1;
20891        runner.state.seats.insert("impl-A".into(), seat);
20892        let winner = runner.state.candidates[0].clone();
20893        runner
20894            .guarded_pr_message(&winner, None, true)
20895            .await
20896            .unwrap()
20897            .expect("rewritten");
20898        assert_eq!(runner.state.seats["impl-A"].turns, 2);
20899        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
20900    }
20901
20902    #[tokio::test]
20903    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
20904        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
20905        // session, was out of quota, and nothing else was asked.
20906        crate::run::pin_test_home();
20907        let dir = tempfile::tempdir().unwrap();
20908        let mut exhausted = spec("alpha");
20909        exhausted.command = vec!["false".into()];
20910        let mut runner = rewrite_runner(
20911            dir.path(),
20912            false,
20913            vec![exhausted, scripted("beta", EN_REWRITE)],
20914        );
20915        // The implementer's own agent is in the role too: it is not asked twice.
20916        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20917            "alpha".into(),
20918            "beta".into(),
20919        ]));
20920        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20921        seat.turns = 1;
20922        runner.state.seats.insert("impl-A".into(), seat);
20923        let winner = runner.state.candidates[0].clone();
20924        let msg = runner
20925            .guarded_pr_message(&winner, None, true)
20926            .await
20927            .unwrap()
20928            .expect("the role rewrites after the implementer fails");
20929        assert!(msg.body.contains("`src/net.rs`"));
20930        assert_eq!(
20931            runner.state.seats["impl-A"].turns, 2,
20932            "the implementer was tried once"
20933        );
20934        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20935        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20936    }
20937
20938    #[tokio::test]
20939    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
20940        crate::run::pin_test_home();
20941        let dir = tempfile::tempdir().unwrap();
20942        let mut broken = spec("bad");
20943        broken.command = vec!["false".into()];
20944        let mut runner = rewrite_runner(
20945            dir.path(),
20946            true,
20947            vec![
20948                broken,
20949                scripted("noformat", "Add retries."),
20950                scripted("good", EN_REWRITE),
20951            ],
20952        );
20953        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20954            "bad".into(),
20955            "noformat".into(),
20956            "good".into(),
20957        ]));
20958        let winner = runner.state.candidates[0].clone();
20959        let msg = runner
20960            .guarded_pr_message(&winner, None, true)
20961            .await
20962            .unwrap()
20963            .expect("the third agent rewrites");
20964        assert!(msg.body.contains("`src/net.rs`"));
20965        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
20966        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
20967    }
20968
20969    #[tokio::test]
20970    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
20971        crate::run::pin_test_home();
20972        // (agents, role): still Japanese; no TITLE; nothing resolvable.
20973        for (agents, role, saved) in [
20974            (
20975                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
20976                Some("beta"),
20977                true,
20978            ),
20979            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
20980            (Vec::new(), None, false),
20981        ] {
20982            let dir = tempfile::tempdir().unwrap();
20983            let mut runner = rewrite_runner(dir.path(), true, agents);
20984            runner.state.config.roles.language_judge =
20985                role.map(|r| crate::config::AgentChoice::One(r.into()));
20986            let winner = runner.state.candidates[0].clone();
20987            let first = runner
20988                .guarded_pr_message(&winner, None, true)
20989                .await
20990                .unwrap();
20991            assert!(first.is_none(), "withheld text parks the run");
20992            assert!(runner.state.parked);
20993            assert_eq!(gate_questions(&runner), 1);
20994            let g = runner.state.github_text.as_ref().unwrap();
20995            assert_eq!(g.rewrite.is_some(), saved);
20996            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
20997        }
20998    }
20999
21000    #[tokio::test]
21001    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
21002        crate::run::pin_test_home();
21003        let dir = tempfile::tempdir().unwrap();
21004        let mut runner = runner_at(RunStatus::Gating);
21005        runner.state =
21006            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
21007        runner.state.repo = dir.path().to_owned();
21008        runner.state.candidates[0].worktree = dir.path().to_owned();
21009        let mut author = spec("alpha");
21010        author.command = vec![
21011            "sh".into(),
21012            "-c".into(),
21013            "printf '%s' \"$REWRITE_REPLY\"".into(),
21014        ];
21015        author.env.insert(
21016            "REWRITE_REPLY".into(),
21017            format!(
21018                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
21019                "English sentence. ".repeat(600)
21020            ),
21021        );
21022        runner.state.config.agents = vec![author];
21023        seed_public_summary(&mut runner);
21024        let winner = runner.state.candidates[0].clone();
21025        assert!(
21026            runner
21027                .guarded_pr_message(&winner, None, true)
21028                .await
21029                .unwrap()
21030                .is_none()
21031        );
21032        let record = runner.state.github_text.clone().unwrap();
21033        let rewrite = record.rewrite.clone().unwrap();
21034        assert_eq!(rewrite.failed, ["body-language"]);
21035        assert!(
21036            rewrite
21037                .body
21038                .ends_with(&format!("magi:run/{}", runner.state.id))
21039        );
21040        let store = ask::Questions::open();
21041        let q = store.get(record.question.as_ref().unwrap()).unwrap();
21042        assert!(
21043            q.choices
21044                .contains(&crate::github_text::USE_REWRITE.to_owned())
21045        );
21046        for label in [
21047            "ORIGINAL",
21048            "FALLBACK",
21049            "REWRITE",
21050            "Rules still failed: body-language",
21051            "truncated",
21052        ] {
21053            assert!(q.detail.contains(label), "{label}");
21054        }
21055        let artifact = std::fs::read_to_string(crate::run::artifact_path(
21056            &runner.state,
21057            &crate::github_text::artifact_name(&record.fingerprint),
21058        ))
21059        .unwrap();
21060        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
21061        // Restart from serialized state, then retry after a bad owner title.
21062        runner.state =
21063            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
21064        store
21065            .update(&q.id, |q| {
21066                q.say("まだ日本語")?;
21067                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
21068            })
21069            .unwrap();
21070        assert!(
21071            runner
21072                .guarded_pr_message(&winner, None, true)
21073                .await
21074                .unwrap()
21075                .is_none()
21076        );
21077        let retry = runner.state.github_text.as_ref().unwrap();
21078        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
21079        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
21080        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
21081        assert!(
21082            q.posting_comparison
21083                .as_ref()
21084                .unwrap()
21085                .matches_detail(&q.detail)
21086        );
21087        store
21088            .update(&q.id, |q| {
21089                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
21090            })
21091            .unwrap();
21092        let message = runner
21093            .guarded_pr_message(&winner, None, true)
21094            .await
21095            .unwrap()
21096            .unwrap();
21097        assert_eq!(message.title, rewrite.title);
21098        assert_eq!(message.body, rewrite.body);
21099        assert_eq!(
21100            runner.state.github_text.as_ref().unwrap().choice,
21101            Some(GithubTextChoice::Rewrite)
21102        );
21103        let again = runner
21104            .guarded_pr_message(&winner, None, true)
21105            .await
21106            .unwrap()
21107            .unwrap();
21108        assert_eq!(again.title, message.title);
21109        assert_eq!(again.body, message.body);
21110        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21111        assert!(
21112            !runner
21113                .state
21114                .events
21115                .iter()
21116                .any(|e| e.message.contains(&rewrite.title)
21117                    || e.message.contains("English sentence."))
21118        );
21119    }
21120
21121    #[tokio::test]
21122    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
21123        crate::run::pin_test_home();
21124        let dir = tempfile::tempdir().unwrap();
21125        let mut runner = runner_at(RunStatus::Gating);
21126        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
21127        runner.state.repo = dir.path().to_owned();
21128        runner.state.candidates[0].worktree = dir.path().to_owned();
21129        seed_public_summary(&mut runner);
21130        let winner = runner.state.candidates[0].clone();
21131        assert!(
21132            runner
21133                .guarded_pr_message(&winner, None, true)
21134                .await
21135                .unwrap()
21136                .is_none()
21137        );
21138        let record = runner.state.github_text.as_ref().unwrap();
21139        assert!(record.rewrite.is_none());
21140        // Records written before the additive field deserialize with no rewrite.
21141        let mut old = serde_json::to_value(record).unwrap();
21142        old.as_object_mut().unwrap().remove("rewrite");
21143        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
21144        assert!(old.rewrite.is_none());
21145        let id = record.question.clone().unwrap();
21146        ask::Questions::open()
21147            .update(&id, |q| {
21148                q.choices.push(crate::github_text::USE_REWRITE.into());
21149                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
21150            })
21151            .unwrap();
21152        let message = runner
21153            .guarded_pr_message(&winner, None, true)
21154            .await
21155            .unwrap()
21156            .unwrap();
21157        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
21158        assert_eq!(
21159            runner.state.github_text.as_ref().unwrap().choice,
21160            Some(GithubTextChoice::Fallback)
21161        );
21162    }
21163
21164    #[tokio::test]
21165    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
21166        crate::run::pin_test_home();
21167        let dir = tempfile::tempdir().unwrap();
21168        let mut runner = runner_at(RunStatus::Gating);
21169        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
21170        runner.state.repo = dir.path().to_owned();
21171        runner.state.candidates[0].worktree = dir.path().to_owned();
21172        seed_public_summary(&mut runner);
21173        let winner = runner.state.candidates[0].clone();
21174        assert!(
21175            runner
21176                .guarded_pr_message(&winner, None, true)
21177                .await
21178                .unwrap()
21179                .is_none()
21180        );
21181        let store = ask::Questions::open();
21182        let find = |run: &str| {
21183            store
21184                .list()
21185                .into_iter()
21186                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
21187                .unwrap()
21188        };
21189        let run = runner.state.id.clone();
21190        let q = find(&run);
21191        let fp = runner
21192            .state
21193            .github_text
21194            .as_ref()
21195            .unwrap()
21196            .fingerprint
21197            .clone();
21198        let name = crate::github_text::artifact_name(&fp);
21199        let path = crate::run::artifact_path(&runner.state, &name);
21200        assert!(q.detail.contains(&path.display().to_string()));
21201        assert!(q.detail.contains("title-language: title"));
21202        let saved = std::fs::read_to_string(&path).expect("artifact written");
21203        assert!(saved.contains("日本語のタイトル"));
21204        assert!(q.cwd.is_none());
21205        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
21206        // A bad replacement is refused and asked about once more.
21207        let say = |id: &str, text: &str, choice: &str| {
21208            store
21209                .update(id, |q| {
21210                    q.thread.push(ask::Turn {
21211                        who: ask::Who::Operator,
21212                        body: text.to_owned(),
21213                        at: jiff::Timestamp::now(),
21214                        note: None,
21215                    });
21216                    q.answer(ask::Answer::Choice(choice.to_owned()))
21217                })
21218                .unwrap();
21219        };
21220        say(&q.id, "まだ日本語", "use my text");
21221        assert!(
21222            runner
21223                .guarded_pr_message(&winner, None, true)
21224                .await
21225                .unwrap()
21226                .is_none()
21227        );
21228        let second = find(&run);
21229        assert_ne!(second.id, q.id);
21230        assert!(second.detail.contains("did not pass"));
21231        // The retry still compares the original text; the rejected
21232        // replacement is named by category only.
21233        assert!(second.detail.contains("日本語のタイトル"));
21234        assert!(!second.detail.contains("まだ日本語"));
21235        assert!(second.detail.contains(&path.display().to_string()));
21236        say(&second.id, "fix: retry failed requests", "use my text");
21237        let message = runner
21238            .guarded_pr_message(&winner, None, true)
21239            .await
21240            .unwrap()
21241            .expect("a vetted replacement is posted");
21242        assert_eq!(message.title, "fix: retry failed requests");
21243        assert_eq!(
21244            runner
21245                .state
21246                .github_text
21247                .as_ref()
21248                .unwrap()
21249                .chosen_title
21250                .as_deref(),
21251            Some("fix: retry failed requests")
21252        );
21253        assert!(runner.state.github_text.as_ref().unwrap().resolved);
21254        // Resumed again: the saved decision is applied, nothing is re-asked.
21255        let again = runner
21256            .guarded_pr_message(&winner, None, true)
21257            .await
21258            .unwrap()
21259            .unwrap();
21260        assert_eq!(again.title, message.title);
21261        assert_eq!(
21262            store
21263                .list()
21264                .into_iter()
21265                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
21266                .count(),
21267            2
21268        );
21269    }
21270
21271    #[test]
21272    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
21273        // What magi itself writes stays English under any configured language,
21274        // so a future localisation of these headings fails here. (The agents'
21275        // own text is also checked by the posting gate.)
21276        let mut state = state_with_summary(
21277            "add retries",
21278            "TITLE: fix(web): batch reads\n- reads run.json once",
21279        );
21280        state.config.graph.language = "ja".to_owned();
21281        let m = pr_message(&state, 'A');
21282        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
21283        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
21284
21285        // The task is the operator's own text: it goes in untouched, and the
21286        // fallback title (no summary) may be in its language too.
21287        let task = "今回やってほしいこと: results projector を直す";
21288        let mut state = state_with_summary(task, "- no title line");
21289        state.config.graph.language = "ja".to_owned();
21290        let m = pr_message(&state, 'A');
21291        assert_eq!(
21292            m.title,
21293            format!("chore: land candidate A of run {}", state.id)
21294        );
21295        assert!(
21296            m.body.contains(&format!(
21297                "<summary>Original task</summary>\n\n{task}\n\n</details>"
21298            )),
21299            "{}",
21300            m.body
21301        );
21302    }
21303
21304    #[test]
21305    fn pr_message_scrubs_home_paths_and_addresses() {
21306        let state = state_with_summary(
21307            "fix it in /Users/someone/src/x",
21308            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
21309        );
21310        let m = pr_message(&state, 'A');
21311        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
21312            assert!(!m.body.contains(leak), "{}", m.body);
21313        }
21314        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
21315    }
21316
21317    #[test]
21318    fn pr_message_survives_a_task_that_closes_details() {
21319        let state = state_with_summary("a </details> b", "TITLE: fix: x");
21320        let m = pr_message(&state, 'A');
21321        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
21322    }
21323
21324    #[test]
21325    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
21326        let cmd = manual_merge_command(
21327            MergeStyle::Squash,
21328            Path::new("/repo"),
21329            "b",
21330            "fix: \"quoted\" $(x) `y`\n\nbody",
21331        );
21332        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
21333    }
21334
21335    #[test]
21336    fn manual_merge_command_matches_the_configured_style() {
21337        let repo = Path::new("/repo");
21338        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
21339
21340        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
21341        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
21342
21343        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
21344        assert_eq!(
21345            squash,
21346            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
21347             \"Merge magi run 0832 (candidate A)\""
21348        );
21349
21350        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
21351        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
21352    }
21353
21354    #[test]
21355    fn a_nudge_gets_a_quarter_of_the_budget() {
21356        // The judge and implement budgets magi ships with.
21357        assert_eq!(retry_budget(secs(1200), true), secs(300));
21358        assert_eq!(retry_budget(secs(3600), true), secs(900));
21359    }
21360
21361    #[test]
21362    fn a_resent_prompt_keeps_the_whole_budget() {
21363        // The seat kept no context, so the retry is the original job again and
21364        // shortening it would only guarantee a second failure.
21365        assert_eq!(retry_budget(secs(1200), false), secs(1200));
21366        assert_eq!(retry_budget(secs(60), false), secs(60));
21367    }
21368
21369    #[test]
21370    fn the_floor_never_exceeds_the_original_budget() {
21371        // A short configured timeout must not be *raised* by the floor: the
21372        // operator asked for a bound, and a retry may not outlast the attempt
21373        // it is retrying.
21374        assert_eq!(retry_budget(secs(60), true), secs(60));
21375        assert_eq!(retry_budget(secs(480), true), secs(120));
21376        assert_eq!(retry_budget(secs(0), true), secs(0));
21377    }
21378
21379    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
21380        agent::CommandEvidence {
21381            id: "item1".to_owned(),
21382            description: "cargo test".to_owned(),
21383            exit_code,
21384            result_summary: String::new(),
21385            source: "codex".to_owned(),
21386        }
21387    }
21388
21389    #[test]
21390    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
21391        // No evidence is not the same fact as unconfirmed evidence: a
21392        // backend with no adapter, or a reply that ran no commands at all,
21393        // must not be misread as carrying a dangling job.
21394        assert!(!has_unconfirmed_command(&[]));
21395    }
21396
21397    #[test]
21398    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
21399        // Deliberately not a check on the exit code's *value*: a fixer
21400        // legitimately runs something that fails mid-iteration before it
21401        // succeeds, and that must never by itself reopen a valid report.
21402        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
21403        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
21404        assert!(!has_unconfirmed_command(&[
21405            evidence(Some(0)),
21406            evidence(Some(101))
21407        ]));
21408    }
21409
21410    #[test]
21411    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
21412        assert!(has_unconfirmed_command(&[
21413            evidence(Some(0)),
21414            evidence(None)
21415        ]));
21416    }
21417
21418    #[test]
21419    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
21420        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21421        assert_eq!(
21422            verified_noop_claim(true, &[], text).as_deref(),
21423            Some("already fixed by b32cfc4, on main.")
21424        );
21425    }
21426
21427    #[test]
21428    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
21429        // A timeout or a bad exit code reads as the ordinary loss it is,
21430        // whatever the reply's own prose claims.
21431        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21432        assert!(verified_noop_claim(false, &[], text).is_none());
21433    }
21434
21435    #[test]
21436    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
21437        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21438        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
21439        // A confirmed command alongside the marker is fine.
21440        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
21441    }
21442
21443    #[test]
21444    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
21445        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
21446    }
21447
21448    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
21449    /// pair, in order, labelled A, B, C, ...
21450    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
21451        runner.state.candidates = shape
21452            .iter()
21453            .enumerate()
21454            .map(|(i, &(empty, verified))| Candidate {
21455                index: i,
21456                label: (b'A' + i as u8) as char,
21457                agent: "sonnet".to_owned(),
21458                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
21459                worktree: PathBuf::from(format!("/wt/{i}")),
21460                summary: String::new(),
21461                stat: String::new(),
21462                files: 0,
21463                commits: 0,
21464                empty,
21465                failed: None,
21466                verified_noop: verified.map(str::to_owned),
21467                duration_ms: 0,
21468                folded: false,
21469            })
21470            .collect();
21471    }
21472
21473    #[test]
21474    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
21475        ask_test_home();
21476        let mut runner = runner_at(RunStatus::Implementing);
21477        set_candidates(
21478            &mut runner,
21479            &[
21480                (true, Some("already on main at b32cfc4")),
21481                (true, Some("same fix, see the existing test")),
21482            ],
21483        );
21484
21485        runner
21486            .after_implement()
21487            .expect("a verified no-op is not an error");
21488
21489        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
21490    }
21491
21492    #[test]
21493    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
21494        ask_test_home();
21495        let mut runner = runner_at(RunStatus::Implementing);
21496        // Candidate A declares a verified no-op; candidate B simply wrote
21497        // nothing and said nothing about why. One candidate's claim is not
21498        // the whole run's agreement.
21499        set_candidates(
21500            &mut runner,
21501            &[(true, Some("already on main at b32cfc4")), (true, None)],
21502        );
21503
21504        let err = runner
21505            .after_implement()
21506            .expect_err("an unverified empty candidate must still fail the run");
21507
21508        assert!(
21509            err.to_string().contains("no candidate produced a change"),
21510            "{err}"
21511        );
21512        assert_eq!(runner.state.status, RunStatus::Failed);
21513    }
21514
21515    #[test]
21516    fn after_implement_still_fails_an_ordinary_all_empty_run() {
21517        ask_test_home();
21518        let mut runner = runner_at(RunStatus::Implementing);
21519        set_candidates(&mut runner, &[(true, None), (true, None)]);
21520
21521        let err = runner
21522            .after_implement()
21523            .expect_err("no candidate declared anything; this is an ordinary failure");
21524
21525        assert!(
21526            err.to_string().contains("no candidate produced a change"),
21527            "{err}"
21528        );
21529        assert_eq!(runner.state.status, RunStatus::Failed);
21530    }
21531}
21532
21533/// A graft rollback whose restore could not be confirmed (see
21534/// [`Runner::graft`]).
21535#[derive(Debug)]
21536struct GraftRestoreFailed(String);
21537
21538impl std::fmt::Display for GraftRestoreFailed {
21539    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
21540        write!(
21541            f,
21542            "could not restore the winner to its pre-graft commit: {}",
21543            self.0
21544        )
21545    }
21546}
21547
21548impl std::error::Error for GraftRestoreFailed {}
21549
21550/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
21551/// not the agent: they never draw on an agent's own timeout, and running out of
21552/// them never reads as an agent failure.
21553#[cfg(test)]
21554mod resource_wait_tests {
21555    use super::*;
21556    use std::collections::BTreeMap;
21557
21558    fn sh(id: &str, script: &str) -> AgentSpec {
21559        AgentSpec {
21560            id: id.to_owned(),
21561            kind: crate::config::AgentKind::Command,
21562            model: None,
21563            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
21564            extra_args: Vec::new(),
21565            env: BTreeMap::new(),
21566            prompt_delivery: None,
21567            max_turn_secs: None,
21568        }
21569    }
21570
21571    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
21572        SeatJob {
21573            seat: SeatState::new("impl-A", &spec.id, 1),
21574            spec,
21575            cwd: dir.to_path_buf(),
21576            prompt: "do it".to_owned(),
21577            timeout,
21578            allow_write: true,
21579            sessions: false,
21580            artifacts: dir.join("art"),
21581            stem: "impl-A".to_owned(),
21582            handover: None,
21583        }
21584    }
21585
21586    fn state(resource_wait_secs: u64) -> RunState {
21587        let mut config = Config::default();
21588        config.graph.timeout_resource_wait = resource_wait_secs;
21589        RunState::new(
21590            PathBuf::from("/repo"),
21591            "main".to_owned(),
21592            "abc1234".to_owned(),
21593            "task".to_owned(),
21594            config,
21595        )
21596    }
21597
21598    fn hold(cache: &Path) -> crate::cache::Guard {
21599        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
21600        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
21601            crate::cache::AcquireOutcome::Acquired(g) => g,
21602            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
21603        }
21604    }
21605
21606    async fn run(
21607        jobs: Vec<SeatJob>,
21608        sem: Arc<Semaphore>,
21609        cache: Option<&Path>,
21610        st: &mut RunState,
21611    ) -> Vec<(usize, SeatState, AgentOutcome)> {
21612        let prompts = Prompts::default();
21613        let ctx = WaveCtx {
21614            run: "r",
21615            node: "implement",
21616            prompts: &prompts,
21617            cache,
21618            round: None,
21619            carry_seats: false,
21620            pauses: None,
21621        };
21622        wave(jobs, sem, &ctx, st, 0).await
21623    }
21624
21625    #[tokio::test]
21626    async fn verification_cache_wait_preserves_the_command_budget() {
21627        crate::run::pin_test_home();
21628        let tmp = tempfile::tempdir().unwrap();
21629        let cache = tmp.path().join("target");
21630        let guard = hold(&cache);
21631        let release = tokio::spawn(async move {
21632            tokio::time::sleep(Duration::from_millis(100)).await;
21633            guard.release();
21634        });
21635        let mut st = state(10);
21636        let budget = Duration::from_millis(50);
21637        let wait = repository_wait(&st, None);
21638        let (outcomes, _) = with_cache_lease(
21639            &mut st,
21640            Some(&cache),
21641            "gate",
21642            "gate",
21643            tmp.path(),
21644            "abc1234",
21645            budget,
21646            "test gate",
21647            &wait,
21648            |_, available| async move {
21649                assert_eq!(
21650                    available, budget,
21651                    "contention must not shorten verification"
21652                );
21653                (Vec::new(), false, Vec::new())
21654            },
21655        )
21656        .await
21657        .unwrap();
21658        release.await.unwrap();
21659        assert!(outcomes.is_empty(), "{outcomes:?}");
21660        assert!(
21661            st.events
21662                .iter()
21663                .any(|event| event.message.contains("waiting for"))
21664        );
21665        assert!(st.resource_wait.is_none());
21666        // The verification body no longer owns the lease after returning.
21667        hold(&cache).release();
21668    }
21669
21670    #[tokio::test]
21671    async fn verification_cache_wait_expires_without_starting_commands() {
21672        crate::run::pin_test_home();
21673        let tmp = tempfile::tempdir().unwrap();
21674        let cache = tmp.path().join("target");
21675        let _held = hold(&cache);
21676        let mut st = state(0);
21677        let wait = repository_wait(&st, None);
21678        let (outcomes, _) = with_cache_lease(
21679            &mut st,
21680            Some(&cache),
21681            "gate",
21682            "gate",
21683            tmp.path(),
21684            "abc1234",
21685            Duration::from_secs(60),
21686            "test gate",
21687            &wait,
21688            |_, _| async { panic!("commands must not start without the cache lease") },
21689        )
21690        .await
21691        .unwrap();
21692        assert_eq!(outcomes.len(), 1);
21693        assert!(outcomes[0].resource_blocked);
21694        assert!(st.resource_wait.is_none());
21695        assert!(matches!(
21696            st.resource_block.unwrap().cause,
21697            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
21698        ));
21699    }
21700
21701    #[tokio::test]
21702    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
21703        crate::run::pin_test_home();
21704        let tmp = tempfile::tempdir().unwrap();
21705        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
21706        let cache = tmp.path().join("cache");
21707        let mut state = state(21600);
21708        let wait = repository_wait(&state, None);
21709        let (outcomes, _) = with_cache_lease(
21710            &mut state,
21711            Some(&cache),
21712            "gate",
21713            "gate",
21714            tmp.path(),
21715            "head",
21716            Duration::from_secs(60),
21717            "gate",
21718            &wait,
21719            |_, _| async { panic!("freshness failure must not start verification") },
21720        )
21721        .await
21722        .unwrap();
21723        assert_eq!(outcomes.len(), 1);
21724        assert!(outcomes[0].resource_blocked);
21725        let block = state.resource_block.unwrap();
21726        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
21727        assert!(!block.description().contains("timed out"));
21728    }
21729
21730    #[tokio::test]
21731    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
21732        crate::run::pin_test_home();
21733        let tmp = tempfile::tempdir().unwrap();
21734        let cache = tmp.path().join("cache");
21735        let _held = hold(&cache);
21736        let mut state = state(21600);
21737        let pause = Pause::new();
21738        let interrupt = Pause::new();
21739        let cause = ParkCause::Operator;
21740        pause.park_for(cause.clone());
21741        let prompts = Prompts::default();
21742        let ctx = WaveCtx {
21743            run: "test",
21744            node: "fix",
21745            prompts: &prompts,
21746            cache: Some(&cache),
21747            round: None,
21748            carry_seats: false,
21749            pauses: Some((&pause, &interrupt)),
21750        };
21751        let result = wave(
21752            vec![job(
21753                sh("writer", "exit 99"),
21754                tmp.path(),
21755                Duration::from_secs(1),
21756            )],
21757            Arc::new(Semaphore::new(1)),
21758            &ctx,
21759            &mut state,
21760            0,
21761        )
21762        .await;
21763        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
21764        assert_eq!(result[0].1.turns, 0);
21765        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21766        assert_eq!(
21767            state.resource_block.unwrap().cause,
21768            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
21769        );
21770        assert!(state.active.is_empty());
21771    }
21772
21773    #[tokio::test]
21774    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
21775        crate::run::pin_test_home();
21776        let tmp = tempfile::tempdir().unwrap();
21777        let cache = tmp.path().join("target-a");
21778        let guard = hold(&cache);
21779        // Held for longer than the agent's whole 1s budget.
21780        tokio::spawn(async move {
21781            tokio::time::sleep(Duration::from_millis(1500)).await;
21782            guard.release();
21783        });
21784        let mut st = state(60);
21785        let results = run(
21786            vec![job(
21787                sh("alpha", "echo ok"),
21788                tmp.path(),
21789                Duration::from_secs(1),
21790            )],
21791            Arc::new(Semaphore::new(1)),
21792            Some(&cache),
21793            &mut st,
21794        )
21795        .await;
21796        assert!(
21797            matches!(results[0].2, AgentOutcome::Ok(_)),
21798            "the wait must not be charged to the agent"
21799        );
21800        assert!(st.resource_block.is_none());
21801        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21802        assert!(st.resource_wait.is_none() && st.active.is_empty());
21803    }
21804
21805    #[tokio::test]
21806    #[cfg(unix)]
21807    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
21808        use std::os::fd::AsRawFd as _;
21809        crate::run::pin_test_home();
21810        let tmp = tempfile::tempdir().unwrap();
21811        crate::git::git(tmp.path(), &["init"]).await.unwrap();
21812        let common = tmp.path().join(".git");
21813        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
21814        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
21815        assert_eq!(
21816            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
21817            0
21818        );
21819        let mut st = state(1);
21820        st.config.coordination = crate::config::test_coordination();
21821        let mut reader = job(
21822            sh("reader", "echo must-not-run"),
21823            tmp.path(),
21824            Duration::from_secs(30),
21825        );
21826        reader.allow_write = false;
21827        let results = run(
21828            vec![reader.clone()],
21829            Arc::new(Semaphore::new(1)),
21830            None,
21831            &mut st,
21832        )
21833        .await;
21834        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21835        assert_eq!(results[0].1.turns, 0);
21836        assert!(FailClass::of(&results[0].2).is_none());
21837        assert_eq!(
21838            st.resource_block.as_ref().unwrap().kind,
21839            "repository coordination"
21840        );
21841        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21842        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21843        drop(external);
21844        let editor = crate::repo_lock::SeatLease::for_checkout(
21845            tmp.path(),
21846            true,
21847            crate::config::test_coordination().as_ref(),
21848        )
21849        .await
21850        .unwrap();
21851        let pause = Pause::new();
21852        let interrupt = Pause::new();
21853        let prompts = Prompts::default();
21854        let ctx = WaveCtx {
21855            run: "r",
21856            node: "review",
21857            prompts: &prompts,
21858            cache: None,
21859            round: None,
21860            carry_seats: false,
21861            pauses: Some((&pause, &interrupt)),
21862        };
21863        let (results, ()) = tokio::join!(
21864            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
21865            async {
21866                tokio::time::sleep(Duration::from_millis(50)).await;
21867                pause.park();
21868            }
21869        );
21870        assert!(
21871            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
21872        );
21873        assert_eq!(results[0].1.turns, 0);
21874        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21875        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21876        drop(editor);
21877    }
21878
21879    #[tokio::test]
21880    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
21881        crate::run::pin_test_home();
21882        let tmp = tempfile::tempdir().unwrap();
21883        let cache = tmp.path().join("target-b");
21884        let _held = hold(&cache);
21885        let mut st = state(1);
21886        let results = run(
21887            vec![job(
21888                sh("alpha", "echo ok"),
21889                tmp.path(),
21890                Duration::from_secs(1),
21891            )],
21892            Arc::new(Semaphore::new(1)),
21893            Some(&cache),
21894            &mut st,
21895        )
21896        .await;
21897        let (_, seat, out) = &results[0];
21898        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
21899        assert!(wave_blocked(&results));
21900        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
21901        assert_eq!(seat.turns, 0, "nothing was asked");
21902        assert!(
21903            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
21904            "no agent was spawned"
21905        );
21906        assert!(FailClass::of(out).is_none(), "not a failure class");
21907        let block = st.resource_block.as_ref().expect("recorded for the resume");
21908        assert_eq!(block.kind, "shared build cache");
21909        assert!(matches!(
21910            block.cause,
21911            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
21912        ));
21913        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21914        assert!(st.active.is_empty());
21915    }
21916
21917    #[tokio::test]
21918    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
21919        crate::run::pin_test_home();
21920        let tmp = tempfile::tempdir().unwrap();
21921        let mut st = state(1);
21922        let results = run(
21923            vec![job(
21924                sh("alpha", "echo ok"),
21925                tmp.path(),
21926                Duration::from_secs(1),
21927            )],
21928            Arc::new(Semaphore::new(0)),
21929            None,
21930            &mut st,
21931        )
21932        .await;
21933        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21934        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
21935        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
21936    }
21937
21938    #[tokio::test]
21939    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
21940        crate::run::pin_test_home();
21941        let tmp = tempfile::tempdir().unwrap();
21942        let mut st = state(60);
21943        let results = run(
21944            vec![job(
21945                sh("alpha", "sleep 30"),
21946                tmp.path(),
21947                Duration::from_secs(1),
21948            )],
21949            Arc::new(Semaphore::new(1)),
21950            None,
21951            &mut st,
21952        )
21953        .await;
21954        let out = &results[0].2;
21955        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
21956        let class = FailClass::of(out).expect("a failure");
21957        assert_eq!(class, FailClass::Timeout);
21958        assert!(should_hand_over(None, &class), "existing handover policy");
21959        assert!(st.resource_block.is_none());
21960    }
21961}