Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
534/// on an early return or a panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615impl Runner {
616    /// Start a fresh run against `repo`.
617    pub async fn start(
618        repo: &Path,
619        instruction: String,
620        config: Config,
621        origin: Origin,
622    ) -> Result<Self> {
623        Self::start_naming(repo, instruction, "", config, origin).await
624    }
625
626    /// [`Runner::start`] for a queued task. Titles and instructions never
627    /// select implementation sources. The title argument is retained for callers.
628    pub async fn start_naming(
629        repo: &Path,
630        instruction: String,
631        _also_scan: &str,
632        config: Config,
633        origin: Origin,
634    ) -> Result<Self> {
635        let repo = git::toplevel(repo).await?;
636        let missing = agent::missing_programs(&config.agents);
637        if !missing.is_empty() {
638            bail!(
639                "these agent programs are not on PATH: {}. Fix the roster in \
640                 magi.toml or install them.",
641                missing.join(", ")
642            );
643        }
644        let base_branch =
645            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
646                .await?;
647        let fetched_base = resolve_base(
648            &repo,
649            &base_branch,
650            &config.merge.remote,
651            config.coordination.as_ref(),
652        )
653        .await?;
654        let base_commit = fetched_base.commit.clone();
655        let roles = config.resolve_roles()?;
656        let max_parallel = config.graph.max_parallel.max(1);
657        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
658        // Recorded before the first save, so a crash right after minting
659        // cannot leave a run with no origin. Legibility only: nothing reads it
660        // to decide anything.
661        state.fetched_base = Some(fetched_base);
662        state.origin = Some(origin);
663        state.reserve()?;
664        state.event("start", format!("run {} created", state.id));
665        state.save()?;
666        Ok(Self {
667            state,
668            roles,
669            sem: Arc::new(Semaphore::new(max_parallel)),
670            pause: Pause::new(),
671            interrupt: Pause::new(),
672        })
673    }
674
675    /// Open a review-only run against work that already exists on `branch`.
676    ///
677    /// The expensive half of the graph is the implement wave — measured at
678    /// 111 and 134 internal tool-loop turns on this repository, against a
679    /// handful for a judge or a reviewer. The cheap half is worth running on
680    /// hand-written work too, and there was no way to reach it.
681    ///
682    /// No new state and no schema change are needed: a run with **one** viable
683    /// candidate and a tally already decided degrades `execute` to exactly
684    /// review → gate → merge, because `judge` skips a single-candidate field,
685    /// `deliberate` has fewer than two first choices to reconcile, `vote`
686    /// returns early, `tally` is already present and `fold_losers` has no
687    /// losers. Resuming such a run therefore does the right thing as well.
688    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
689        Self::review_taking_over(repo, branch, config, None, origin).await
690    }
691
692    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
693    /// at the same task still has `branch` checked out, its worktree is
694    /// released first if that is safe (see [`crate::handover`]), and the
695    /// review refuses with the reason if it is not. `None` is a hand-run
696    /// review: it has no earlier attempts, so only a worktree of a dead run
697    /// magi recorded itself can be released.
698    pub async fn review_taking_over(
699        repo: &Path,
700        branch: &str,
701        config: Config,
702        takeover: Option<crate::handover::Takeover>,
703        origin: Origin,
704    ) -> Result<Self> {
705        let repo = git::toplevel(repo).await?;
706        let missing = agent::missing_programs(&config.agents);
707        if !missing.is_empty() {
708            bail!(
709                "these agent programs are not on PATH: {}. Fix the roster in \
710                 magi.toml or install them.",
711                missing.join(", ")
712            );
713        }
714        let base_branch =
715            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
716                .await?;
717        if base_branch == branch {
718            bail!("`{branch}` is the base branch; there is nothing to review against");
719        }
720        let fetched_base = resolve_base(
721            &repo,
722            &base_branch,
723            &config.merge.remote,
724            config.coordination.as_ref(),
725        )
726        .await?;
727        let base_commit = fetched_base.commit.clone();
728
729        let roles = config.resolve_roles()?;
730        let max_parallel = config.graph.max_parallel.max(1);
731        let mut state = RunState::new(
732            repo.clone(),
733            base_branch,
734            base_commit.clone(),
735            String::new(),
736            config,
737        );
738        state.fetched_base = Some(fetched_base);
739        state.origin = Some(origin);
740        state.reserve()?;
741
742        // Released before anything else touches the branch: a stale local
743        // branch is moved with `git branch -f`, which git refuses while an
744        // earlier attempt's worktree still has it checked out. Everything
745        // after this point that can fail puts the old run back.
746        // A hand-run review has no task, hence no earlier attempts, but a
747        // worktree of a dead run magi made may still be released.
748        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
749            earlier: Vec::new(),
750            home: crate::run::home(),
751            choice: None,
752            task_instruction: None,
753        });
754        let wait = repository_wait(&state, None);
755        let coordination_config = state.config.coordination.clone();
756        let coordination =
757            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
758                .await?;
759        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
760        if let Some(released) = &released {
761            state.event(
762                "release",
763                format!(
764                    "took `{branch}` over from run {}: its worktree was released: {}",
765                    crate::run::short_of(&released.old_id),
766                    released.audit
767                ),
768            );
769        }
770        // The owner's answer to an earlier divergence question is applied
771        // here: after the release (git will not move a checked-out branch)
772        // and before the sync that would otherwise ask again.
773        state.foreign_checkout = crate::handover::preflight_review(
774            &repo,
775            branch,
776            &takeover.home,
777            &state.config.merge.remote,
778        )
779        .await?;
780        if state.foreign_checkout.is_some() && takeover.choice.is_some() {
781            bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
782        }
783        if let Some(choice) = takeover.choice.as_ref()
784            && let Err(e) =
785                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
786                    .await
787        {
788            if let Some(released) = &released {
789                released.restore(&repo, branch).await;
790            }
791            return Err(e.context("applying the owner's answer about the diverged branch"));
792        }
793        // An earlier attempt of this task that published the task head, and
794        // whose work is the branch under review, hands that head over: the
795        // review run updates the same pull request. Without one (a legacy
796        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
797        // own head and its own open pull request is adopted as before.
798        for id in takeover.earlier.iter().rev() {
799            if let Ok(old) = RunState::load_under(id, &takeover.home)
800                && let Some(head) = old.pr_head.clone()
801                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
802            {
803                state.pr_head = Some(head);
804                break;
805            }
806        }
807        drop(coordination);
808        let opened = Self::open_review(
809            &repo,
810            branch,
811            state,
812            roles,
813            max_parallel,
814            base_commit,
815            &takeover,
816        )
817        .await;
818        if opened.is_err()
819            && let Some(released) = &released
820        {
821            let _coordination = crate::repo_lock::SeatLease::operation_with(
822                &repo,
823                coordination_config.as_ref(),
824                &wait,
825            )
826            .await?;
827            released.restore(&repo, branch).await;
828        }
829        opened
830    }
831
832    /// Recover exactly the operator-selected public implementation, without touching old sources.
833    pub async fn recover_pr(
834        repo: &Path,
835        task: &str,
836        selection: crate::pr_recovery::Selection,
837        mut config: Config,
838    ) -> Result<Self> {
839        selection.configure(&mut config)?;
840        selection.check_task(task, &selection.instruction)?;
841        let base_branch =
842            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
843                .await?;
844        let fetched_base = resolve_base(
845            repo,
846            &base_branch,
847            &config.merge.remote,
848            config.coordination.as_ref(),
849        )
850        .await?;
851        let base_commit = fetched_base.commit.clone();
852        selection
853            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
854            .await?;
855        Box::pin(selection.verify_source(repo)).await?;
856        if let Some(proof) = &selection.source_proof {
857            anyhow::ensure!(
858                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
859                "source rebase base is outside the fetched landing base"
860            );
861        }
862        let roles = config.resolve_roles()?;
863        let max_parallel = config.graph.max_parallel.max(1);
864        let mut state = RunState::new(
865            repo.to_path_buf(),
866            base_branch,
867            base_commit.clone(),
868            selection.instruction.clone(),
869            config,
870        );
871        state.fetched_base = Some(fetched_base);
872        state.origin = Some(crate::run::Origin::queue(task));
873        state.pr_head = Some(selection.branch.clone());
874        state.pr_recovery = Some(selection.clone());
875        state.reserve()?;
876        if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
877            // Never leave a reserved record behind as a phantom owner of the head.
878            state.discard_reserved();
879            return Err(e);
880        }
881        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
882    }
883
884    async fn open_selected(
885        repo: &Path,
886        state: RunState,
887        roles: ResolvedRoles,
888        max_parallel: usize,
889        base_commit: String,
890        selection: crate::pr_recovery::Selection,
891    ) -> Result<Self> {
892        let mut state = state;
893        let branch = state.branch_for('A');
894        let wait = repository_wait(&state, None);
895        let prepared = async {
896            let coordination = coordinate_operation(&mut state, repo, &wait).await?;
897            git::git(repo, &["branch", &branch, selection.source()]).await?;
898            drop(coordination);
899            anyhow::Ok(())
900        }
901        .await;
902        if let Err(e) = prepared {
903            // Nothing was exposed yet: do not leave a reserved record as a
904            // phantom owner of the public head.
905            state.discard_reserved();
906            return Err(e);
907        }
908        state.event(
909            "recovery",
910            format!(
911                "selected implementation {} for {} (public lease {})",
912                selection.source(),
913                selection.url,
914                selection.head
915            ),
916        );
917        let takeover = crate::handover::Takeover {
918            earlier: Vec::new(),
919            home: crate::run::home(),
920            choice: None,
921            task_instruction: Some(selection.instruction.clone()),
922        };
923        if let Err(e) = state.save() {
924            state.discard_reserved();
925            return Err(e);
926        }
927        let id = state.id.clone();
928        let opened = Self::open_review(
929            repo,
930            &branch,
931            state,
932            roles,
933            max_parallel,
934            base_commit,
935            &takeover,
936        )
937        .await;
938        match opened {
939            Ok(runner) => Ok(runner),
940            Err(e) => {
941                if let Ok(mut partial) = RunState::load(&id) {
942                    partial.status = RunStatus::Blocked;
943                    partial.driver_exited = true;
944                    partial.event(
945                        "recovery",
946                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
947                    );
948                    partial.save()?;
949                }
950                Err(crate::pr_recovery::StartupFailed {
951                    run: id,
952                    cause: format!("{e:#}"),
953                }
954                .into())
955            }
956        }
957    }
958
959    /// What the newest earlier attempt that reviewed `branch` leaves for this
960    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
961    /// no earlier attempts. Reads the earlier record only: ownership of its
962    /// worktree and branch is [`crate::handover`]'s business.
963    async fn inherit(
964        repo: &Path,
965        branch: &str,
966        base_commit: &str,
967        takeover: &crate::handover::Takeover,
968    ) -> Option<crate::recovery::RecoveryRecord> {
969        use crate::recovery::{Relation, assess};
970        let newest = takeover.earlier.last()?;
971        let mut chosen = None;
972        for id in takeover.earlier.iter().rev() {
973            if let Ok(old) = RunState::load_under(id, &takeover.home)
974                && !old.reviews.is_empty()
975                && (old.winner().is_some_and(|w| w.branch == branch)
976                    || old.pr_head.as_deref() == Some(branch))
977            {
978                chosen = Some(old);
979                break;
980            }
981        }
982        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
983            .await
984            .unwrap_or_default();
985        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
986            Some(last) if last.head == tip => Relation::Same,
987            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
988            _ => Relation::Diverged,
989        };
990        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
991        Some(assess(from, chosen.as_ref(), base_commit, relation))
992    }
993
994    /// The half of [`Runner::review_taking_over`] that can fail after an
995    /// earlier attempt's worktree was released.
996    async fn open_review(
997        repo: &Path,
998        branch: &str,
999        mut state: RunState,
1000        roles: ResolvedRoles,
1001        max_parallel: usize,
1002        base_commit: String,
1003        takeover: &crate::handover::Takeover,
1004    ) -> Result<Self> {
1005        let wait = repository_wait(&state, None);
1006        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1007        if state.foreign_checkout.is_some() {
1008            // Never reconcile or force-move a ref checked out by the operator.
1009            if state.pr_head.is_none() {
1010                state.pr_head = Some(branch.to_owned());
1011            }
1012        } else if state.pr_recovery.is_none() {
1013            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1014        }
1015        // The commit subjects are the closest thing to a task statement that
1016        // existing work carries, and the reviewers are told as much.
1017        let start = review_base(
1018            repo,
1019            &state.config.merge.remote,
1020            &state.base_branch,
1021            &base_commit,
1022            branch,
1023        )
1024        .await;
1025        let log = git::log_oneline(repo, &start, branch)
1026            .await
1027            .unwrap_or_default();
1028        let log_text = if log.trim().is_empty() {
1029            "(no commit messages)"
1030        } else {
1031            log.trim()
1032        };
1033        let mut instruction = format!(
1034            "Review the work already on branch `{branch}`. There is no task \
1035             statement: what the change claims to do is whatever its commits \
1036             say.\n\n{log_text}"
1037        );
1038        // A retry of a task whose earlier attempt left reviews behind carries
1039        // that attempt's evidence, and the task's own statement with it: the
1040        // completion criteria are the task's, not whatever the commits claim.
1041        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1042            if let Some(task) = takeover
1043                .task_instruction
1044                .as_deref()
1045                .filter(|t| !t.trim().is_empty())
1046                && rec.reviewed_head.is_some()
1047            {
1048                instruction = format!(
1049                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1050                    task.trim()
1051                );
1052                rec.criteria_inherited = true;
1053            }
1054            state.event("recovery", rec.summary());
1055            state.recovery = Some(rec);
1056        }
1057        if let Some(selected) = &state.pr_recovery {
1058            instruction = selected.instruction.clone();
1059        }
1060        state.task_instruction = takeover.task_instruction.clone();
1061        state.instruction = instruction;
1062        state.reviewed_commits = Some(
1063            git::subjects(repo, &start, branch)
1064                .await
1065                .unwrap_or_default(),
1066        );
1067
1068        // A takeover must not forget a hold that a previous review observed,
1069        // even if repairs have since removed it from the branch.
1070        for id in &takeover.earlier {
1071            if let Ok(old) = RunState::load_under(id, &takeover.home)
1072                && old.repo == state.repo
1073                && (old.pr_head.as_deref() == Some(branch)
1074                    || old.winner().is_some_and(|w| w.branch == branch))
1075            {
1076                for hold in old.branch_owner_holds {
1077                    if !state.branch_owner_holds.contains(&hold) {
1078                        state.event(
1079                            "approval-policy",
1080                            format!(
1081                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1082                                hold.control.key(),
1083                                hold.file,
1084                                hold.head
1085                            ),
1086                        );
1087                        state.branch_owner_holds.push(hold);
1088                    }
1089                }
1090            }
1091        }
1092        // Select the actual review tip before observing owner policy. A foreign
1093        // checkout can lag its remote descendant, and repairs must not erase
1094        // holds introduced only on that selected commit.
1095        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1096            let tracking = format!("{}/{branch}", state.config.merge.remote);
1097            // A surviving tracking ref is not evidence that the remote still
1098            // offers this tip. Adopt it only after a successful fetch here.
1099            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1100            let mut head = checkout.synced_head.clone();
1101            if matches!(fetched, Ok(out) if out.ok()) {
1102                let remote = git::rev_parse(repo, &tracking).await?;
1103                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1104                    head = remote;
1105                }
1106            }
1107            head
1108        } else {
1109            git::rev_parse(repo, branch).await?
1110        };
1111        state.observe_branch_owner_holds(&reviewed_head).await;
1112
1113        // An attached worktree, so the fixer's commits land on the branch under
1114        // review rather than on a detached head nobody will look at again.
1115        let worktree = state.worktree_root().join("under-review");
1116        if let Some(parent) = worktree.parent() {
1117            tokio::fs::create_dir_all(parent).await.ok();
1118        }
1119        let path = worktree.to_string_lossy().to_string();
1120        let review_branch = if state.foreign_checkout.is_some() {
1121            let shadow = format!("magi/{}/review", state.short());
1122            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1123            shadow
1124        } else {
1125            branch.to_owned()
1126        };
1127        git::git(repo, &["worktree", "add", &path, &review_branch])
1128            .await
1129            .with_context(|| {
1130                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1131            })?;
1132
1133        let setup = match ensure_setup_config(&mut state, repo).await {
1134            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1135            Err(e) => Err(e),
1136        };
1137        if let Err(e) = setup {
1138            git::worktree_remove_clean(repo, &worktree).await.ok();
1139            return Err(e);
1140        }
1141        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1142            .await
1143            .unwrap_or(0);
1144        if commits == 0 {
1145            git::worktree_remove_clean(repo, &worktree).await.ok();
1146            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1147        }
1148        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1149            .await
1150            .map(|f| f.len())
1151            .unwrap_or(0);
1152        if files == 0
1153            && let (Ok(head_tree), Ok(base_tree)) = (
1154                git::tree_of(&worktree, "HEAD").await,
1155                git::tree_of(&worktree, &base_commit).await,
1156            )
1157            && head_tree == base_tree
1158        {
1159            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1160            git::worktree_remove_clean(repo, &worktree).await.ok();
1161            bail!(
1162                "`{branch}` at {} has a tree identical to base {}; this usually means \
1163                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1164                 against `{}/{branch}`) rather than an empty change",
1165                short(&head),
1166                short(&base_commit),
1167                state.config.merge.remote
1168            );
1169        }
1170        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1171            .await
1172            .unwrap_or_default();
1173
1174        state.candidates.push(Candidate {
1175            index: 0,
1176            label: 'A',
1177            // Not an agent id on purpose: nothing in the roster wrote this, and
1178            // the stats tables must not credit anyone with a win for it.
1179            agent: EXISTING_BRANCH.to_owned(),
1180            branch: review_branch,
1181            worktree,
1182            summary: String::new(),
1183            stat,
1184            files,
1185            commits,
1186            empty: false,
1187            failed: None,
1188            verified_noop: None,
1189            duration_ms: 0,
1190            folded: false,
1191        });
1192        state.tally = Some(Tally {
1193            first_choice: BTreeMap::from([('A', 0)]),
1194            borda: BTreeMap::new(),
1195            winner: 'A',
1196            rankings: 0,
1197            unanimous_initial: false,
1198            deliberated: false,
1199            changed_votes: 0,
1200            unanimous_final: false,
1201            tie_break: None,
1202            // No panel sat, so no quorum applies. Zero judges is the correct
1203            // number for work that never competed, and must not be reported as
1204            // a collapsed panel.
1205            judges: 0,
1206            present: 0,
1207            quorum: 0,
1208            met_quorum: true,
1209            uncontested: Some("review-only run: nothing competed".to_owned()),
1210        });
1211        state.status = RunStatus::Reviewing;
1212        state.prep_complete = Some(true);
1213        state.event(
1214            "start",
1215            format!(
1216                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1217                state.id
1218            ),
1219        );
1220        state.save()?;
1221        Ok(Self {
1222            state,
1223            roles,
1224            sem: Arc::new(Semaphore::new(max_parallel)),
1225            pause: Pause::new(),
1226            interrupt: Pause::new(),
1227        })
1228    }
1229
1230    /// Authorize exactly one additional rebase, under the caller's queue task
1231    /// claim and the same owner/execution locks used by run maintenance.
1232    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1233        Self::authorize_base_recovery_request(home, id, task, false).await
1234    }
1235
1236    /// Queue release uses its durable approval generation exactly once.
1237    pub(crate) async fn authorize_base_recovery_from_resumption(
1238        home: &Path,
1239        id: &str,
1240        task: &str,
1241    ) -> Result<()> {
1242        Self::authorize_base_recovery_request(home, id, task, true).await
1243    }
1244
1245    async fn authorize_base_recovery_request(
1246        home: &Path,
1247        id: &str,
1248        task: &str,
1249        resumption: bool,
1250    ) -> Result<()> {
1251        let _owner = crate::pr_head::owner_lock(home, id)?;
1252        let mut state = RunState::load_under(id, home)?;
1253        crate::pr_head::ensure_owner(&state)?;
1254        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1255        anyhow::ensure!(
1256            queued.runs.last().is_some_and(|run| run == id)
1257                && queued.status != crate::queue::TaskStatus::Done
1258                && !queued.fresh_start
1259                && queued.review_branch.is_none(),
1260            "run task ownership changed"
1261        );
1262        anyhow::ensure!(
1263            state
1264                .origin
1265                .as_ref()
1266                .and_then(|o| o.task.as_deref())
1267                .is_none_or(|owner| owner == task),
1268            "run belongs to another task"
1269        );
1270        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1271        anyhow::ensure!(
1272            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1273            "run is live or its ownership is unknown"
1274        );
1275        // Judge the previous driver's lease before acquiring our maintenance
1276        // lease. The owner lock excludes new driver claims across both steps.
1277        let _execution = crate::driver::maintenance(home, id)?;
1278        anyhow::ensure!(
1279            state.status.resumable()
1280                && state.released_to.is_none()
1281                && state.retired_to.is_none()
1282                && state.pr_taken_over_by.is_none(),
1283            "run ownership is final"
1284        );
1285        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1286        anyhow::ensure!(
1287            !state.pr_publication_pending
1288                && !crate::handover::landing_checkpoint(&state)
1289                && state.merge_decision.is_none(),
1290            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1291        );
1292        let winner = state.winner().context("run has no retained winner")?;
1293        anyhow::ensure!(
1294            git::is_clean(&winner.worktree).await?,
1295            "retained worktree has uncommitted changes"
1296        );
1297        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1298        anyhow::ensure!(
1299            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1300            "retained worktree does not match the winner branch"
1301        );
1302        state.record_base_sync_stop(&head);
1303        state.save()?;
1304        if resumption {
1305            anyhow::ensure!(
1306                crate::daemon::compatible_run_instruction(&state, &queued),
1307                "resumption instruction does not match the retained run"
1308            );
1309            let approval = queued
1310                .resumption
1311                .as_ref()
1312                .context("no operator resumption")?;
1313            anyhow::ensure!(
1314                approval.instruction == queued.instruction,
1315                "resumption approval does not match the current task instruction"
1316            );
1317            state.authorize_base_sync_resumption(task, approval)?;
1318        } else {
1319            state.authorize_base_sync(task)?;
1320        }
1321        state.event(
1322            "base-recovery",
1323            "owner authorized one additional base rebase; fresh verification required",
1324        );
1325        state.save()?;
1326        Ok(())
1327    }
1328
1329    /// Reopen an existing run.
1330    pub fn resume(id: &str) -> Result<Self> {
1331        let state = RunState::load(id)?;
1332        crate::pr_head::ensure_owner(&state)?;
1333        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1334            bail!(
1335                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1336            );
1337        }
1338        if let Some(to) = &state.released_to {
1339            bail!(
1340                "run {} cannot be resumed: its worktree was released to run {}",
1341                state.short(),
1342                crate::run::short_of(to)
1343            );
1344        }
1345        let roles = state.config.resolve_roles()?;
1346        if let Some(id) = &state.config.graph.selected_implementer
1347            && state.candidates.iter().any(|c| c.agent != *id)
1348        {
1349            bail!(
1350                "replacement implementer pin does not match the saved candidates; task must stay held"
1351            );
1352        }
1353        let max_parallel = state.config.graph.max_parallel.max(1);
1354        Ok(Self {
1355            state,
1356            roles,
1357            sem: Arc::new(Semaphore::new(max_parallel)),
1358            pause: Pause::new(),
1359            interrupt: Pause::new(),
1360        })
1361    }
1362
1363    /// Walk the graph to a terminal state, skipping nodes already recorded.
1364    ///
1365    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1366    /// the phone - ends here, so this is the one place a run that ended
1367    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1368    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1369    pub async fn execute(&mut self) -> Result<()> {
1370        if let Some(selection) = &self.state.pr_recovery
1371            && (selection.source_proof.is_some() || selection.source_run.is_some())
1372        {
1373            Box::pin(selection.verify_source(&self.state.repo)).await?;
1374        }
1375        // Queue claims cover daemon work; this also serializes manual drivers
1376        // consuming a repair answer, including the new review and decision.
1377        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1378            || self.state.land_approval.is_some()
1379            || self.state.merge_decision.is_some()
1380            || !self.state.merge_repairs.is_empty()
1381            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1382        {
1383            let claim = FixClaim::acquire(&self.state.dir())?;
1384            let disk = RunState::load(&self.state.id)?;
1385            let checkpoint = |s: &RunState| {
1386                (
1387                    s.merge_repairs.len(),
1388                    s.merge_repairs
1389                        .last()
1390                        .map(|r| (r.question.clone(), r.phase)),
1391                    s.land_approval
1392                        .as_ref()
1393                        .map(|a| (a.question.clone(), a.head.clone())),
1394                )
1395            };
1396            if checkpoint(&disk) != checkpoint(&self.state) {
1397                bail!(
1398                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1399                );
1400            }
1401            Some(claim)
1402        } else {
1403            None
1404        };
1405        let mut execution = self.claim_driver().await?;
1406        let runner = &mut *execution.runner;
1407        let wait = runner.repository_wait();
1408        // Keep the graph's large future off the callers' stack, including
1409        // standalone task attempts running on a default-sized test thread.
1410        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1411        let result = match result {
1412            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1413                record_repository_block(&mut runner.state, &e);
1414                runner.park_here()?;
1415                Ok(())
1416            }
1417            other => other,
1418        };
1419        execution.driver.finish(&mut runner.state).await?;
1420        let ended = if result.is_err() {
1421            Some({
1422                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1423                if let Err(error) = &result {
1424                    notice.message = format!(
1425                        "Run {} stopped with an error: {}",
1426                        runner.state.id,
1427                        error.to_string().lines().next().unwrap_or("unknown error")
1428                    );
1429                }
1430                notice
1431            })
1432        } else {
1433            crate::notices::run_ended(&runner.state)
1434        };
1435        if let Some(notice) = ended {
1436            crate::notices::raise(notice);
1437        }
1438        result
1439    }
1440
1441    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1442        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1443    }
1444
1445    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1446        let wait = self.repository_wait();
1447        coordinate_operation(&mut self.state, cwd, &wait).await
1448    }
1449
1450    /// Validate ownership and persist the driver claim in one critical section.
1451    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1452        let owner = match crate::pr_head::owner_lock_wait(
1453            &crate::run::home(),
1454            &self.state.id,
1455            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1456        )
1457        .await
1458        {
1459            Ok(owner) => owner,
1460            Err(error) => {
1461                let detail = format!("worktree maintenance or ownership update: {error:#}");
1462                // Admission did not own run.json: retain this resource outcome
1463                // in memory for queue settlement without overwriting a concurrent
1464                // retirement or publication transaction.
1465                self.state.resource_block = Some(crate::run::ResourceBlock {
1466                    cause: crate::run::ResourceBlockCause::Other,
1467                    node: "start".into(),
1468                    kind: "worktree maintenance".into(),
1469                    detail: detail.clone(),
1470                    at: Timestamp::now(),
1471                });
1472                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1473            }
1474        };
1475        let disk = RunState::load(&self.state.id)?;
1476        crate::pr_head::ensure_owner(&disk)?;
1477        let claimed =
1478            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1479        // Only proven-live ownership refuses here. Unknown records (legacy runs
1480        // with no claim marker or process identity) fall through to the
1481        // exclusive execution lease taken just below, which is what actually
1482        // keeps a second driver out.
1483        anyhow::ensure!(
1484            disk.liveness(claimed) != crate::run::Liveness::Live,
1485            "run already has a live driver; wait for its execution to end"
1486        );
1487        if let Some(to) = &disk.released_to {
1488            bail!(
1489                "run {} cannot continue: its worktree was released to run {}",
1490                self.state.short(),
1491                crate::run::short_of(to)
1492            );
1493        }
1494        anyhow::ensure!(
1495            serde_json::to_value(&disk.base_sync_recovery)?
1496                == serde_json::to_value(&self.state.base_sync_recovery)?,
1497            "base recovery changed since this run was loaded; reload the saved run"
1498        );
1499        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1500        let mut execution = DriverExecution {
1501            runner: self,
1502            driver,
1503        };
1504        // Declare after the execution guard: on any claim error this short
1505        // lock must drop before that guard reloads and releases ownership.
1506        let _owner = owner;
1507        let state = &mut execution.runner.state;
1508        // Moving again, so it is no longer parked. Set before the walk rather
1509        // than in `resume`, so every way of re-entering the graph clears it
1510        // and a card cannot claim a run is waiting to be resumed while the
1511        // agents are already working.
1512        state.parked = false;
1513        state.resource_wait = None;
1514        state.resource_block = None;
1515        // Any seat this state still lists as answering belongs to whatever
1516        // process last drove this run — this one included, if it crashed
1517        // mid-wave. Cleared and flushed immediately, before anything else
1518        // runs, so a resume can never show a seat as live when nothing is
1519        // asking it anything yet; the node that actually dispatches the next
1520        // wave repopulates it.
1521        state.clear_active();
1522        // Recorded in the same spot, and flushed together with the clear
1523        // above: this is the pid a reader checks (`RunState::liveness`) when
1524        // no daemon claim exists to answer "is a process still driving this
1525        // run" — a plain `magi run` / `magi review` typed into a terminal
1526        // claims nothing there. Always overwritten, never only-if-absent, so
1527        // a resumed run's stale pid from a previous, possibly-dead process
1528        // can never survive into this one's own report. Unlike
1529        // `clear_active`, this changes on every single `execute()` call, so
1530        // the save below is now unconditional rather than only-if-cleared.
1531        //
1532        // `driver_started_at` is recorded in the same breath, from this same
1533        // pid, so `liveness` can tell a live pid that is genuinely still us
1534        // apart from one the OS has since handed to an unrelated process —
1535        // see that field's own doc for why the pid alone is not enough.
1536        let pid = std::process::id();
1537        state.driver_pid = Some(pid);
1538        state.driver_started_at = crate::proc::process_started_at(pid);
1539        state.driver_exited = false;
1540        state.driver_claimed = Some(true);
1541        state.begin_pass();
1542        execution.driver.bind(state);
1543        state.save()?;
1544        execution.driver.record(state)?;
1545        Ok(execution)
1546    }
1547
1548    async fn execute_graph(&mut self) -> Result<()> {
1549        self.state.configure_recovery_resume()?;
1550        // Preservation may invalidate an old empty classification. Resolve it
1551        // before any resume shortcut can reuse a verdict or publish a branch.
1552        self.persist_candidates().await?;
1553        // A saved base stop outranks stage markers, including legacy Voting.
1554        // Recovery never reenters implementation, deliberation or voting.
1555        if self.state.base_sync_blocked() {
1556            self.sync_to_base().await?;
1557            if !self.state.base_sync_blocked()
1558                && self.state.status != RunStatus::AlreadyInBase
1559                && !self.resource_halt()?
1560            {
1561                self.finish_after_tally().await?;
1562            }
1563            return Ok(());
1564        }
1565        if self
1566            .state
1567            .merge_decision
1568            .as_ref()
1569            .is_some_and(|decision| !decision.closed)
1570            && self.state.status != RunStatus::Landing
1571        {
1572            if crate::merge_decision::resumable_hold(&self.state) {
1573                self.state.merge = Some(MergeOutcome {
1574                    mode: MergeMode::Pr,
1575                    ok: false,
1576                    detail: self
1577                        .state
1578                        .merge_decision
1579                        .as_ref()
1580                        .unwrap()
1581                        .target
1582                        .pr
1583                        .clone(),
1584                    empty: false,
1585                });
1586                self.run_land().await?;
1587                self.settle_questions();
1588                return Ok(());
1589            }
1590            bail!(
1591                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1592            );
1593        }
1594        if self.state.config.graph.selected_implementer.is_some()
1595            && self.state.status == RunStatus::Blocked
1596            && self.state.tally.is_none()
1597            && self.state.candidate_preservation.is_empty()
1598            && self.state.interrupted_implementations.is_empty()
1599        {
1600            bail!(
1601                "selected implementation is held; explicitly release the owning task to retry it"
1602            );
1603        }
1604
1605        if self.state.merge_repairs.last().is_some_and(|r| {
1606            matches!(
1607                r.phase,
1608                crate::run::MergeRepairPhase::Requested
1609                    | crate::run::MergeRepairPhase::Fixing
1610                    | crate::run::MergeRepairPhase::Reviewing
1611            )
1612        }) {
1613            if self.repair_merge_review().await? {
1614                self.run_land().await?;
1615            }
1616            self.settle_questions();
1617            return Ok(());
1618        }
1619        // Publication already passed review and gate. Resume only that node:
1620        // syncing to a newer base here could change the verified commit.
1621        if self.state.pr_publication_pending {
1622            self.merge().await?;
1623            if self.resource_halt()? {
1624                return Ok(());
1625            }
1626            self.state.save()?;
1627            self.settle_questions();
1628            return Ok(());
1629        }
1630        // A run that already lost its quorum never resumes into the verdict
1631        // machinery: `deliberate` and `vote` would otherwise clobber the
1632        // stalled marker back to Voting and the run would keep going past a
1633        // verdict that is no longer trustworthy. Everything already recorded is
1634        // kept, so the run stays resumable (or foldable) for a human to pick up.
1635        //
1636        // On --resume the run gets one chance to repair itself: the seats a
1637        // rate limit took out are re-asked. If their quota has since reset and
1638        // the quorum is restored, the run picks up and finishes; otherwise it
1639        // stays stale and still-resumable for a later retry. If it does not
1640        // recover, the returned status stays `Stalled` and nothing was
1641        // clobbered (the recovery only mutates entries for the lost seats).
1642        if self.state.status == RunStatus::Stalled {
1643            if self.recover_stall().await? {
1644                self.finish_after_tally().await?;
1645            } else {
1646                // Still below quorum: persist the marker and stay resumable.
1647                self.state.save()?;
1648            }
1649            return Ok(());
1650        }
1651        // A run parked inside `land` - watching CI, mid fix-round, or
1652        // waiting on the owner's merge approval - resumes directly into it,
1653        // never back through `prep`. Everything before `merge` already
1654        // concluded; that is the only way `status` reaches `Landing` in the
1655        // first place. Re-walking `review_loop` first would also be actively
1656        // wrong: its own status recomputation (see its doc) treats any
1657        // clean round as reason to set `status` to `Gating`, which would
1658        // clobber this marker before `merge` ever ran, and this run would
1659        // never find its way back into `land` at all.
1660        if self.state.status == RunStatus::Landing
1661            || crate::handover::landing_checkpoint(&self.state)
1662        {
1663            self.state.land_sync_pending = true;
1664            self.state.save()?;
1665            self.run_land().await?;
1666            // `run_land` may have settled the run right here - CI came back
1667            // green and the PR merged, say - without ever passing back
1668            // through `merge`'s own trailing call. Whatever it left `status`
1669            // as is what this has to read.
1670            self.settle_questions();
1671            return Ok(());
1672        }
1673        // A run parked at `merge` on the owner's word about a withheld PR text
1674        // resumes straight into `merge`. Re-walking the tail would sync to the
1675        // base again (possibly rebasing, possibly spending the fixer) after the
1676        // gate had passed, and then push a tree nobody reviewed or gated.
1677        if self.state.github_text.is_some()
1678            && self.state.merge.is_none()
1679            && self.state.status == RunStatus::Gating
1680        {
1681            self.merge().await?;
1682            if self.resource_halt()? {
1683                return Ok(());
1684            }
1685            self.state.save()?;
1686            return Ok(());
1687        }
1688        if let Err(e) = self.prep().await {
1689            if e.is::<crate::repo_lock::ResourceBlocked>() {
1690                return Err(e);
1691            }
1692            self.state.status = RunStatus::Blocked;
1693            self.state
1694                .prep_failure
1695                .get_or_insert_with(|| format!("{e:#}"));
1696            self.state
1697                .event("prep", format!("preparation stopped: {e:#}"));
1698            self.state.save()?;
1699            return Err(e);
1700        }
1701        if self.park_here()? {
1702            return Ok(());
1703        }
1704        self.advise().await?;
1705        if self.park_here()? {
1706            return Ok(());
1707        }
1708        self.implement().await?;
1709        if self.park_here()? {
1710            return Ok(());
1711        }
1712        // `after_implement` already saved the state and settled any open
1713        // questions when it set this; nothing later in the graph has
1714        // anything to judge.
1715        if self.state.status == RunStatus::VerifiedNoop {
1716            return Ok(());
1717        }
1718        self.judge().await?;
1719        if self.park_here()? {
1720            return Ok(());
1721        }
1722        self.deliberate().await?;
1723        if self.park_here()? {
1724            return Ok(());
1725        }
1726        self.vote().await?;
1727        if self.park_here()? {
1728            return Ok(());
1729        }
1730        self.tally()?;
1731        // A verdict that lost its quorum is not trustworthy: do not review,
1732        // gate, or merge on it. Everything already done is kept, so the run
1733        // stays resumable (or foldable); the human can replace the agent that
1734        // ran out of quota and pick it up.
1735        if self.state.status == RunStatus::Stalled {
1736            // Persist the stalled marker now — the normal end-of-execute save
1737            // below is below this early return, and without it a resumed run
1738            // would reload a pre-tally status and keep going.
1739            self.state.save()?;
1740            return Ok(());
1741        }
1742        self.finish_after_tally().await?;
1743        Ok(())
1744    }
1745
1746    /// A wave could not start its agents because a shared resource stayed busy
1747    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1748    /// resumable. Unstarted seats have no result; completed peers retain
1749    /// their saved candidate state.
1750    fn stop_for_resource(&mut self) -> Result<()> {
1751        // Resource acquisition observes cancellation inside a node. Complete
1752        // the same durable park as every other node boundary before returning.
1753        if self.pause.parked() || self.interrupt.parked() {
1754            self.park_here()?;
1755            return Ok(());
1756        }
1757        if self.state.resource_block.is_some() {
1758            self.state.status = RunStatus::Blocked;
1759        }
1760        self.state.save()
1761    }
1762
1763    /// Stop walking the graph if a node ended on a resource block.
1764    fn resource_halt(&mut self) -> Result<bool> {
1765        if self.state.resource_block.is_none() {
1766            return Ok(false);
1767        }
1768        self.stop_for_resource()?;
1769        Ok(true)
1770    }
1771
1772    /// Park here if asked to, recording it in the run's own timeline.
1773    ///
1774    /// Returns whether the caller should stop walking the graph. The state is
1775    /// saved either way by the node that just finished; this adds the event so
1776    /// the operator's card says why a run that is neither finished nor moving
1777    /// is sitting where it is.
1778    fn park_here(&mut self) -> Result<bool> {
1779        let parking = self.pause.parked() || self.interrupt.parked();
1780        if !parking && self.state.resource_block.is_some() {
1781            self.stop_for_resource()?;
1782            return Ok(true);
1783        }
1784        // Either handle asking is enough - see `Pause`'s own doc for why
1785        // they are never the same one. `interrupt` is checked second so a
1786        // reason it carries is preferred in the message below over a plain
1787        // shutdown park racing it at the same boundary.
1788        if !self.pause.parked() && !self.interrupt.parked() {
1789            return Ok(false);
1790        }
1791        if self.state.parked {
1792            return Ok(true);
1793        }
1794        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1795        if let Some(block) = &mut self.state.resource_block
1796            && matches!(
1797                block.cause,
1798                crate::run::ResourceBlockCause::Cancelled { .. }
1799            )
1800        {
1801            block.cause = crate::run::ResourceBlockCause::Cancelled {
1802                park: cause.clone(),
1803            };
1804        }
1805        let why = match cause.as_ref().map(ParkCause::label) {
1806            Some(reason) => format!(
1807                "parked after `{}` ({reason}) — resume to carry on from here",
1808                self.state.status.as_str()
1809            ),
1810            None => format!(
1811                "parked after `{}` — resume to carry on from here",
1812                self.state.status.as_str()
1813            ),
1814        };
1815        self.state.event("park", why);
1816        self.state
1817            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1818        self.state.parked = true;
1819        self.state.save()?;
1820        Ok(true)
1821    }
1822
1823    /// Only a new explicit operator approval may retry failed preparation.
1824    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1825        if self.state.prep_failure.is_none() {
1826            return Ok(());
1827        }
1828        let failed_at = self
1829            .state
1830            .events
1831            .iter()
1832            .rev()
1833            .find(|event| event.node == "prep")
1834            .map_or(self.state.updated_at, |event| event.at);
1835        anyhow::ensure!(
1836            approval_at > failed_at,
1837            "preparation retry requires a new explicit release after the failure"
1838        );
1839        self.state.prep_failure = None;
1840        self.state.prep_complete = Some(false);
1841        self.state
1842            .event("prep-retry", "operator approved retry of preparation");
1843        Ok(())
1844    }
1845
1846    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1847    pub fn on_pause(&mut self, pause: Pause) {
1848        self.pause = pause;
1849    }
1850
1851    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1852    /// scheduler asking this one run - and no other - to park so a task
1853    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1854    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1855    /// handle.
1856    pub fn watch_interrupt(&mut self, pause: Pause) {
1857        self.interrupt = pause;
1858    }
1859
1860    /// Abandon this run's own open questions, once `status` has actually
1861    /// settled rather than merely paused.
1862    ///
1863    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1864    /// either back up with the candidates, the review round and the seat
1865    /// sessions already on disk, so a question an implementer asked mid-round
1866    /// may still get a real answer read by a real resume. Only the statuses
1867    /// `resumable` excludes are actually final: the run merged, it reached
1868    /// `Ready` with nothing left to do, it failed outright with no
1869    /// established point to continue from, or every candidate agreed, with
1870    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1871    /// every one of those the seat that asked is gone for good, exactly like
1872    /// the run being deleted under `magi run rm` - so the same cleanup
1873    /// applies, worded for what actually happened instead of "the run was
1874    /// deleted".
1875    ///
1876    /// Best-effort and silent on success: called from every place `status`
1877    /// can land on one of those three, including ones a resumed run revisits,
1878    /// so it must cost nothing when there was nothing open to begin with.
1879    fn settle_questions(&mut self) {
1880        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1881            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1882        }
1883    }
1884
1885    /// The tail of the graph after a trustworthy tally: fold losers, review,
1886    /// gate, merge, and persist.
1887    async fn finish_after_tally(&mut self) -> Result<()> {
1888        // Before the losers are folded: their branches are what the graft
1889        // reads from.
1890        self.graft().await?;
1891        if self.resource_halt()? {
1892            return Ok(());
1893        }
1894        self.fold_losers().await?;
1895        // Before review starts, and again right before the gate: a run's
1896        // review rounds can themselves take long enough for the base to move
1897        // a second time, and the gate is the one node whose "green" gets
1898        // acted on.
1899        self.sync_to_base().await?;
1900        if self.hold_base_sync().await? || self.resource_halt()? {
1901            return Ok(());
1902        }
1903        if self.state.status == RunStatus::AlreadyInBase {
1904            return Ok(());
1905        }
1906        self.review_loop().await?;
1907        if self.resource_halt()? {
1908            return Ok(());
1909        }
1910        self.sync_to_base().await?;
1911        if self.hold_base_sync().await? || self.resource_halt()? {
1912            return Ok(());
1913        }
1914        if self.state.status == RunStatus::AlreadyInBase {
1915            return Ok(());
1916        }
1917        self.gate().await?;
1918        self.merge().await?;
1919        if self.resource_halt()? {
1920            return Ok(());
1921        }
1922        self.state.save()?;
1923        Ok(())
1924    }
1925
1926    // ---------------------------------------------------------------- prep
1927
1928    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1929    /// run with the step and its output in the event; it is never swallowed.
1930    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1931    /// resume starts over instead of finding candidates and missing seats.
1932    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1933        let result = match ensure_setup_config(&mut self.state, repo).await {
1934            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1935            Err(e) => Err(e),
1936        };
1937        if let Err(e) = result {
1938            git::worktree_remove_clean(repo, wt).await.ok();
1939            if in_prep {
1940                self.rollback_prep(repo).await;
1941            }
1942            self.state.status = RunStatus::Blocked;
1943            self.state.event(
1944                "setup",
1945                format!("worktree setup failed in {}: {e:#}", wt.display()),
1946            );
1947            self.state.save()?;
1948            return Err(e);
1949        }
1950        Ok(())
1951    }
1952
1953    /// Undo everything `prep` created, so a resume re-runs it whole.
1954    async fn rollback_prep(&mut self, repo: &Path) {
1955        let root = self.state.worktree_root();
1956        for c in std::mem::take(&mut self.state.candidates) {
1957            if matches!(
1958                git::worktree_remove_clean(repo, &c.worktree).await,
1959                Ok(true)
1960            ) {
1961                git::branch_delete(repo, &c.branch).await.ok();
1962            } else {
1963                self.state.candidates.push(c);
1964            }
1965        }
1966        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1967        for j in 1..=seats {
1968            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1969                .await
1970                .ok();
1971        }
1972        for k in 1..=self.state.config.graph.advisors {
1973            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1974                .await
1975                .ok();
1976        }
1977    }
1978
1979    async fn prep(&mut self) -> Result<()> {
1980        if let Some(reason) = &self.state.prep_failure {
1981            bail!(
1982                "candidate preparation previously failed: {reason}; correct the source before retrying"
1983            );
1984        }
1985        if self
1986            .state
1987            .prep_complete
1988            .unwrap_or(!self.state.candidates.is_empty())
1989        {
1990            return Ok(());
1991        }
1992        let repo = self.state.repo.clone();
1993        // Includes rollback, config, refs, checkout creation and setup. No
1994        // repository mutation may precede the real shared lock acquisition.
1995        let _coordination = self.coordinate(&repo).await?;
1996        let base = self.state.base_commit.clone();
1997        let plan = self.state.sources.plan(&repo, &base).await?;
1998        self.rollback_prep(&repo).await;
1999        if !self.state.candidates.is_empty() {
2000            bail!(
2001                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2002            );
2003        }
2004        self.state.prep_complete = Some(false);
2005        self.state.status = RunStatus::Prep;
2006        self.state.save()?;
2007        let repo = self.state.repo.clone();
2008        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2009        let root = self.state.worktree_root();
2010        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2011
2012        // The hook is the write-time half of the blindness contract; the
2013        // presentation filter in `blind` is the half that cannot be bypassed.
2014        let hooks_dir = self.state.dir().join("hooks");
2015        if self.state.config.blind.commit_msg_hook {
2016            std::fs::create_dir_all(&hooks_dir)
2017                .with_context(|| format!("create {}", hooks_dir.display()))?;
2018            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2019            let path = hooks_dir.join("commit-msg");
2020            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2021            make_executable(&path)?;
2022            // Ref-counted rather than a plain idempotent set: with more than
2023            // one run able to be in flight in the same repository at once
2024            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2025            // true?" check cannot tell "another run of mine still needs
2026            // this" from "nobody does", and the run that happens to finish
2027            // first would disable the hook out from under a sibling still
2028            // relying on it.
2029            git::acquire_worktree_config(&repo).await?;
2030            self.state.enabled_worktree_config = true;
2031        }
2032
2033        // `[worktree] setup` hides its products per worktree, which needs the
2034        // same per-worktree config the hook does. Held until fold, like it.
2035        ensure_setup_config(&mut self.state, &repo).await?;
2036
2037        for (index, (spec, label)) in self
2038            .roles
2039            .implementers
2040            .clone()
2041            .into_iter()
2042            .zip(labels)
2043            .enumerate()
2044        {
2045            let branch = self.state.branch_for(label);
2046            let worktree = root.join(format!("cand-{label}"));
2047            // Persist intent before Git can leave a partially created tree.
2048            // Candidate registration follows setup, so it cannot inventory
2049            // crashes or refusals during creation itself.
2050            self.state.event(
2051                "prep-intent",
2052                serde_json::json!({
2053                    "branch": branch,
2054                    "worktree": worktree,
2055                    "start": start,
2056                    "picks": plan.picks,
2057                })
2058                .to_string(),
2059            );
2060            self.state.save()?;
2061            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2062            if self.state.config.blind.commit_msg_hook {
2063                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2064            }
2065            git::local_exclude(&worktree, "/.magi/").await?;
2066            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2067                // Nothing of this prep may survive: a resume must start over,
2068                // not find half the candidates and skip the rest.
2069                git::worktree_remove_clean(&repo, &worktree).await.ok();
2070                git::branch_delete(&repo, &branch).await.ok();
2071                self.rollback_prep(&repo).await;
2072                self.state.status = RunStatus::Blocked;
2073                self.state
2074                    .event("prep", format!("worktree setup failed: {e:#}"));
2075                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2076                self.state.save()?;
2077                return Err(e);
2078            }
2079            self.state.candidates.push(Candidate {
2080                index,
2081                label,
2082                agent: spec.id.clone(),
2083                branch,
2084                worktree,
2085                summary: String::new(),
2086                stat: String::new(),
2087                files: 0,
2088                commits: 0,
2089                empty: false,
2090                failed: None,
2091                verified_noop: None,
2092                duration_ms: 0,
2093                folded: false,
2094            });
2095            self.state.save()?;
2096            for pick in &plan.picks {
2097                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2098                {
2099                    self.state.status = RunStatus::Blocked;
2100                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2101                    self.state
2102                        .event("prep", format!("cannot apply referenced commit: {e}"));
2103                    self.state.save()?;
2104                    return Err(e);
2105                }
2106            }
2107        }
2108
2109        for j in 1..=self.roles.judges.len() {
2110            let wt = root.join(format!("judge-{j}"));
2111            if !wt.exists() {
2112                git::worktree_add_detached(&repo, &wt, &base).await?;
2113                self.setup_seat_worktree(&repo, &wt, true).await?;
2114            }
2115        }
2116
2117        // Disposable, detached checkouts for the design-deliberation stage's
2118        // advisor seats — the same shape as the judges' above, at the same
2119        // base commit, since advisors also only ever read. Sized off the
2120        // configured count directly rather than a resolved roster: unlike
2121        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2122        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2123        // `prep` has no `ResolvedRoles` field to read a count from here.
2124        if self.state.config.graph.advise {
2125            for k in 1..=self.state.config.graph.advisors {
2126                let wt = root.join(format!("advisor-{k}"));
2127                if !wt.exists() {
2128                    git::worktree_add_detached(&repo, &wt, &base).await?;
2129                    self.setup_seat_worktree(&repo, &wt, true).await?;
2130                }
2131            }
2132        }
2133
2134        // A judge cannot tell it is looking at its own patch — the seats keep
2135        // separate conversations — but a panel that shares agents with the
2136        // field is less independent than it looks, and that is worth saying out
2137        // loud once per run rather than leaving it in the config.
2138        let authors: Vec<&str> = self
2139            .roles
2140            .implementers
2141            .iter()
2142            .map(|a| a.id.as_str())
2143            .collect();
2144        let overlap: Vec<String> = self
2145            .roles
2146            .judges
2147            .iter()
2148            .enumerate()
2149            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2150            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2151            .collect();
2152        if !overlap.is_empty() {
2153            let note = format!(
2154                "{} also authored a candidate; blind, but the panel is less \
2155                 independent than {} distinct agents would be",
2156                overlap.join(", "),
2157                self.roles.judges.len()
2158            );
2159            self.state.event("prep", note);
2160        }
2161
2162        self.state.event(
2163            "prep",
2164            format!(
2165                "{} candidates, {} judges, base {} ({})",
2166                self.state.candidates.len(),
2167                self.roles.judges.len(),
2168                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2169                self.state.base_branch
2170            ),
2171        );
2172        self.state.prep_complete = Some(true);
2173        self.state.status = RunStatus::Implementing;
2174        self.state.save()?;
2175        Ok(())
2176    }
2177
2178    // -------------------------------------------------------------- advise
2179
2180    /// The design-deliberation stage: independent, read-only advisor seats
2181    /// each sketch a design before any implementer touches the repository,
2182    /// and (when at least one produced a usable proposal) a synthesis seat
2183    /// blends them into a brief `implement` carries in every candidate's
2184    /// prompt.
2185    ///
2186    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2187    /// advisors` is the proposal count. Everything here is best-effort and
2188    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2189    /// that cannot reach quota, or a synthesis seat that produced nothing
2190    /// usable all leave `implement` exactly as it was before this stage
2191    /// existed — the task instruction alone — rather than failing the whole
2192    /// competition over an enrichment stage. Every outcome is still recorded
2193    /// as an event, so a run that got nothing from this stage says why.
2194    ///
2195    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2196    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2197    /// resumed run whose stage failed would re-run it, and re-spend the
2198    /// agent calls, on every reentry before `implement`.
2199    ///
2200    /// Also skipped once any candidate shows implementation progress — the
2201    /// exact predicate `implement` itself uses to decide a candidate is no
2202    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2203    /// is not enough: a run created by an older binary that predates this
2204    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2205    /// an already-`Implementing`-or-later run under this build would
2206    /// otherwise walk straight back through `prep` (a no-op once candidates
2207    /// exist) into this node and spawn every advisor seat against worktrees
2208    /// `prep` never recreated — after implementation has already started,
2209    /// which is exactly the invariant this stage exists to guarantee.
2210    async fn advise(&mut self) -> Result<()> {
2211        let implement_untouched = self
2212            .state
2213            .candidates
2214            .iter()
2215            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2216        if !self.state.config.graph.advise || self.state.advise_attempted {
2217            return Ok(());
2218        }
2219        if !implement_untouched {
2220            self.state.event(
2221                "advise",
2222                "skipping the design-deliberation stage: at least one \
2223                 candidate already shows implementation progress, so this \
2224                 run is past the point the stage exists to run before"
2225                    .to_owned(),
2226            );
2227            self.state.advise_attempted = true;
2228            self.state.save()?;
2229            return Ok(());
2230        }
2231        let run_id = self.state.id.clone();
2232        let prompts = self.state.config.prompts.clone();
2233        let instruction = self.state.instruction.clone();
2234        let language = self.state.config.graph.language.clone();
2235        let root = self.state.worktree_root();
2236        let n = self.state.config.graph.advisors;
2237        let where_recorded = self.state.dir().join("run.json");
2238
2239        let seats = match self.state.config.advisors() {
2240            Ok(seats) if !seats.is_empty() => seats,
2241            Ok(_) => {
2242                self.state.event(
2243                    "advise",
2244                    format!(
2245                        "[graph] advisors is 0; skipping the design-deliberation \
2246                         stage and continuing without a synthesis brief (see {})",
2247                        where_recorded.display()
2248                    ),
2249                );
2250                self.state.advise_attempted = true;
2251                self.state.save()?;
2252                return Ok(());
2253            }
2254            Err(e) => {
2255                self.state.event(
2256                    "advise",
2257                    format!(
2258                        "could not resolve advisor seats ({e:#}); continuing \
2259                         without a design-deliberation brief (see {})",
2260                        where_recorded.display()
2261                    ),
2262                );
2263                self.state.advise_attempted = true;
2264                self.state.save()?;
2265                return Ok(());
2266            }
2267        };
2268
2269        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2270        let artifacts = agent::artifacts_dir(&self.state.dir());
2271        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2272
2273        let mut jobs = Vec::new();
2274        for (i, spec) in seats.iter().cloned().enumerate() {
2275            let seat_key = format!("advisor-{}", i + 1);
2276            let seat = self.seat(&seat_key, &spec.id);
2277            jobs.push(SeatJob {
2278                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2279                spec,
2280                seat,
2281                cwd: worktrees[i % worktrees.len()].clone(),
2282                timeout,
2283                allow_write: false,
2284                sessions: false,
2285                artifacts: artifacts.clone(),
2286                stem: seat_key,
2287                handover: None,
2288            });
2289        }
2290
2291        self.state.event(
2292            "advise",
2293            format!(
2294                "{} advisor seat(s) sketching a design in parallel",
2295                jobs.len()
2296            ),
2297        );
2298        let mut quota_losses = Vec::new();
2299        let cache = self.state.config.cache_dir();
2300        let ctx = WaveCtx {
2301            carry_seats: false,
2302            pauses: Some((&self.pause, &self.interrupt)),
2303            run: &run_id,
2304            node: "advise",
2305            prompts: &prompts,
2306            cache: cache.as_deref(),
2307            round: None,
2308        };
2309        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2310        let results = ask_json_wave::<Proposal>(
2311            jobs,
2312            Arc::clone(&self.sem),
2313            self.state.config.graph.retries,
2314            &advisor_roster,
2315            &ctx,
2316            &mut quota_losses,
2317            &mut self.state,
2318            &|p: &Proposal| p.validate(),
2319        )
2320        .await;
2321        if self.state.resource_block.is_some() {
2322            return self.stop_for_resource();
2323        }
2324        self.state.quota.extend(quota_losses);
2325
2326        let mut records = Vec::with_capacity(results.len());
2327        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2328            let agent_id = seat.agent.clone();
2329            self.state.seats.insert(seat.key.clone(), seat);
2330            match res {
2331                Ok((proposal, out)) => {
2332                    self.state
2333                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2334                    records.push(advise::AdvisorRecord::proposed(
2335                        i + 1,
2336                        agent_id,
2337                        proposal,
2338                        out.duration_ms,
2339                    ));
2340                }
2341                Err(e) => {
2342                    self.state.event(
2343                        "advise",
2344                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2345                    );
2346                    records.push(advise::AdvisorRecord::failed(
2347                        i + 1,
2348                        agent_id,
2349                        e.to_string(),
2350                    ));
2351                }
2352            }
2353        }
2354
2355        let mut advice = advise::Advice {
2356            records,
2357            synthesis: None,
2358        };
2359        if advice.proposals().is_empty() {
2360            self.state.event(
2361                "advise",
2362                "no advisor produced a usable proposal; continuing without a \
2363                 synthesis brief"
2364                    .to_owned(),
2365            );
2366        } else {
2367            match self
2368                .synthesize_brief(
2369                    &advice,
2370                    &instruction,
2371                    &language,
2372                    &worktrees[0],
2373                    &artifacts,
2374                    &run_id,
2375                    &prompts,
2376                    cache.as_deref(),
2377                )
2378                .await
2379            {
2380                Ok(Some(text)) => {
2381                    self.state.event(
2382                        "advise",
2383                        "synthesized a design brief for the implementer".to_owned(),
2384                    );
2385                    advice.synthesis = Some(text);
2386                }
2387                Ok(None) => {
2388                    self.state.event(
2389                        "advise",
2390                        "the synthesis seat produced nothing usable; continuing \
2391                         without a design brief"
2392                            .to_owned(),
2393                    );
2394                }
2395                Err(e) => {
2396                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2397                        return Err(e);
2398                    }
2399                    self.state.event(
2400                        "advise",
2401                        format!("could not synthesize a design brief: {e:#}"),
2402                    );
2403                }
2404            }
2405        }
2406        advise::apply_reflection(&mut advice);
2407
2408        self.state.advice = Some(advice);
2409        self.state.advise_attempted = true;
2410        self.state.save()?;
2411        Ok(())
2412    }
2413
2414    /// The synthesis seat: reads every advisor's proposal and blends them
2415    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2416    /// out of [`Runner::advise`] only for readability — it is not called
2417    /// anywhere else.
2418    ///
2419    /// Picked the same way [`crate::talk`]'s standing conversation and
2420    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2421    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2422    /// order (a claude seat, else the first runnable agent in roster order)
2423    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2424    /// in [`crate::config`] for why a dedicated field exists here at all.
2425    #[allow(clippy::too_many_arguments)]
2426    async fn synthesize_brief(
2427        &mut self,
2428        advice: &advise::Advice,
2429        instruction: &str,
2430        language: &str,
2431        cwd: &Path,
2432        artifacts: &Path,
2433        run_id: &str,
2434        prompts: &Prompts,
2435        cache: Option<&Path>,
2436    ) -> Result<Option<String>> {
2437        let chain = agent::pick_chain(
2438            &self.state.config.agents,
2439            self.state.config.roles.synthesizer.as_ref(),
2440            &agent::installed,
2441            "synthesizer",
2442        )?;
2443        let proposals = advice.proposals();
2444        let mut prompt = prompt::with_overlay(
2445            prompt::synthesize_brief(instruction, &proposals, language),
2446            prompts.overlay("advise"),
2447        );
2448        if cache.is_some() {
2449            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2450            // below — see `prompt::build_cache_note`'s doc for why telling a
2451            // read-only seat to build through the shared cache is exactly how
2452            // a sandbox's write refusal gets misread as a defect.
2453            prompt.push('\n');
2454            prompt.push_str(&prompt::build_cache_note("advise", false));
2455        }
2456        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2457        // Each id is tried once, in order; a quota hit, error or unusable
2458        // answer moves to the next. The seat is single-turn (`sessions:
2459        // false`) and the prompt is the whole context, so a fallback agent
2460        // needs nothing carried over.
2461        let mut last = None;
2462        for (n, spec) in chain.iter().enumerate() {
2463            if n > 0 {
2464                self.state
2465                    .event("advise", format!("synthesis falling back to {}", spec.id));
2466            }
2467            let mut seat = self.seat("advise-synthesis", &spec.id);
2468            let wait = self.repository_wait();
2469            let outcome = wait
2470                .scope(agent::invoke(
2471                    spec,
2472                    &mut seat,
2473                    &Invocation {
2474                        coordination: self.state.config.coordination.as_ref(),
2475                        fetched_base: None,
2476                        cwd,
2477                        prompt: &prompt,
2478                        timeout,
2479                        allow_write: false,
2480                        sessions: false,
2481                        artifacts,
2482                        stem: &if n == 0 {
2483                            "advise-synthesis".to_owned()
2484                        } else {
2485                            format!("advise-synthesis-{}", spec.id)
2486                        },
2487                        run: run_id,
2488                        node: "advise",
2489                        cache_dir: None,
2490                        attachments: &[],
2491                        writable: &[],
2492                    },
2493                ))
2494                .await;
2495            if outcome.is_ok() {
2496                self.state.seats.insert(seat.key.clone(), seat);
2497            }
2498            let advance = agent::chain_advances(&outcome);
2499            last = Some(outcome);
2500            if !advance {
2501                break;
2502            }
2503        }
2504        // Exhausted: the last attempt's result is what a single failed seat
2505        // would have produced.
2506        let out = last.expect("a chain holds at least one agent")?;
2507        if !out.usable() {
2508            return Ok(None);
2509        }
2510        let text =
2511            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2512        Ok((!text.trim().is_empty()).then_some(text))
2513    }
2514
2515    // ----------------------------------------------------------- implement
2516
2517    async fn implement(&mut self) -> Result<()> {
2518        self.persist_candidates().await?;
2519        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2520        // agent files with `magi task add` name the run that paid for it. The
2521        // prompt overlay is cloned alongside it because the waves borrow it
2522        // while `self` is mutably borrowed by the node's own bookkeeping.
2523        let run_id = self.state.id.clone();
2524        let prompts = self.state.config.prompts.clone();
2525        let todo: Vec<usize> = self
2526            .state
2527            .candidates
2528            .iter()
2529            .enumerate()
2530            .filter(|(_, c)| {
2531                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2532                    && c.failed.is_none()
2533                    && !c.empty
2534            })
2535            .map(|(i, _)| i)
2536            .collect();
2537        if todo.is_empty() {
2538            return self.after_implement();
2539        }
2540        self.state.status = RunStatus::Implementing;
2541
2542        let language = self.state.config.graph.language.clone();
2543        let github_language = self.state.config.graph.github_language.clone();
2544        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2545        let sessions = self.state.config.graph.sessions;
2546        let artifacts = agent::artifacts_dir(&self.state.dir());
2547        // The design-deliberation stage's blended brief, when `advise` found
2548        // one — carried into every implementer's prompt the same way
2549        // regardless of which candidate it is.
2550        let brief = self
2551            .state
2552            .advice
2553            .as_ref()
2554            .and_then(|a| a.synthesis.as_deref())
2555            .map(str::to_owned);
2556        let attachments = self.state.attachments.clone();
2557
2558        let mut jobs = Vec::new();
2559        for &i in &todo {
2560            let (index, label, worktree) = {
2561                let c = &self.state.candidates[i];
2562                (c.index, c.label, c.worktree.clone())
2563            };
2564            let spec = self.roles.implementers[index].clone();
2565            let seat_key = format!("impl-{label}");
2566            let seat = self.seat(&seat_key, &spec.id);
2567            let instruction = seeded_instruction(&self.state);
2568            jobs.push(SeatJob {
2569                spec,
2570                seat,
2571                prompt: prompt::implement_in(
2572                    &instruction,
2573                    &worktree.to_string_lossy(),
2574                    &language,
2575                    &github_language,
2576                    brief.as_deref(),
2577                    &attachments,
2578                ),
2579                cwd: worktree,
2580                timeout,
2581                allow_write: true,
2582                sessions,
2583                artifacts: artifacts.clone(),
2584                stem: format!("impl-{label}"),
2585                handover: None,
2586            });
2587        }
2588
2589        self.state.event(
2590            "implement",
2591            format!("{} candidates in parallel", jobs.len()),
2592        );
2593        // Kept so a seat whose CLI hung up can be asked again from the same
2594        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2595        // can update a seat's own entry once a fallback agent takes it over —
2596        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2597        // whichever agent actually answered, not the one that quota'd out.
2598        let mut sent = jobs.clone();
2599        let cache = self.state.config.cache_dir();
2600        let ctx = WaveCtx {
2601            carry_seats: false,
2602            pauses: Some((&self.pause, &self.interrupt)),
2603            run: &run_id,
2604            node: "implement",
2605            prompts: &prompts,
2606            cache: cache.as_deref(),
2607            round: None,
2608        };
2609        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2610        if !wave_blocked(&results) {
2611            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2612                .await;
2613            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2614                .await?;
2615            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2616                .await;
2617        }
2618        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2619
2620        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2621            && results
2622                .iter()
2623                .any(|(_, _, out)| FailClass::of(out).is_some());
2624        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2625            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2626                // Executed turns interrupted by disk pressure already carry
2627                // a preservation barrier from wave. Admission-only waits do
2628                // not. Keep the interrupted seat's session for recovery too.
2629                if self
2630                    .state
2631                    .candidate_preservation
2632                    .contains_key(&self.state.candidates[i].label)
2633                {
2634                    self.state.seats.insert(seat.key.clone(), seat);
2635                }
2636                continue;
2637            }
2638            let seat_key = seat.key.clone();
2639            // A quota fallback (`resume_seat_handovers`) may have handed this
2640            // seat to a different agent than the one `prep` recorded on the
2641            // candidate; the stats tables and any later fixer-defaults-to-
2642            // winner's-author lookup must credit whoever actually answered —
2643            // unless every fallback also quota'd out, in which case nobody
2644            // actually answered and crediting the last agent tried would
2645            // erase every earlier agent's own quota loss from the stats
2646            // tables instead of just this one seat's.
2647            let agent = seat.agent.clone();
2648            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2649            self.state.seats.insert(seat.key.clone(), seat);
2650            let label = self.state.candidates[i].label;
2651            self.state.interrupted_implementations.remove(&label);
2652
2653            let (summary, duration, failed, verified_claim) = match out {
2654                AgentOutcome::Ok(o) => {
2655                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2656                    let failed = (!o.usable()).then(|| {
2657                        if o.timed_out {
2658                            "agent timed out".to_owned()
2659                        } else {
2660                            format!(
2661                                "agent exited with {}",
2662                                crate::notices::exit_status(o.exit_code)
2663                            )
2664                        }
2665                    });
2666                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2667                    (text, o.duration_ms, failed, verified_claim)
2668                }
2669                // Left un-resumed by `resume_undelivered` (a dirty tree
2670                // already rescues the work, or there was no session left to
2671                // resume into) — reported like the ordinary failure it is,
2672                // never as if `o.text` (the CLI's raw error JSON) were an
2673                // answer.
2674                AgentOutcome::Dropped(o) => {
2675                    let why = o
2676                        .dropped
2677                        .as_ref()
2678                        .map(|d| d.why.as_str())
2679                        .unwrap_or("the CLI ended the stream without delivering its answer");
2680                    (
2681                        String::new(),
2682                        o.duration_ms,
2683                        Some(format!("the CLI dropped the stream ({why})")),
2684                        None,
2685                    )
2686                }
2687                AgentOutcome::Quota(o) => {
2688                    self.state.quota.push(QuotaLoss {
2689                        seat: seat_key,
2690                        node: "implement".to_owned(),
2691                        at: Timestamp::now(),
2692                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2693                    });
2694                    (
2695                        String::new(),
2696                        o.duration_ms,
2697                        Some("rate limited (quota); produced no change".to_owned()),
2698                        None,
2699                    )
2700                }
2701                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2702                    (String::new(), 0, Some(e), None)
2703                }
2704            };
2705
2706            // Save every completed turn before any fallible Git operation. A failed
2707            // rescue must not erase the reply or prevent collection of other seats.
2708            let c = &mut self.state.candidates[i];
2709            if !exhausted_the_fallback_chain {
2710                c.agent = agent;
2711            }
2712            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2713            c.duration_ms = duration;
2714            c.failed = failed;
2715            c.verified_noop = verified_claim;
2716            self.state
2717                .candidate_preservation
2718                .insert(label, "awaiting persistence".to_owned());
2719        }
2720        self.state.save()?;
2721        self.persist_candidates().await?;
2722
2723        if resource_blocked {
2724            return self.stop_for_resource();
2725        }
2726        if selected_failed {
2727            self.state.status = RunStatus::Blocked;
2728            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2729            self.state.save()?;
2730            bail!(
2731                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2732            );
2733        }
2734        self.after_implement()
2735    }
2736
2737    /// Retry only persistence after a completed implementation turn.
2738    async fn persist_candidates(&mut self) -> Result<()> {
2739        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2740        for label in labels {
2741            let i = self
2742                .state
2743                .candidates
2744                .iter()
2745                .position(|c| c.label == label)
2746                .context("pending candidate is missing")?;
2747            let c = self.state.candidates[i].clone();
2748            let interrupted = self.state.interrupted_implementations.contains(&label);
2749            let result = async {
2750                let wait = crate::repo_lock::WaitPolicy::new(
2751                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2752                    || false,
2753                );
2754                let _coordination =
2755                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2756                // Best effort evidence requires no index writes. A successful snapshot
2757                // never authorizes removal of a tree whose rescue failed.
2758                if let Err(e) = git::snapshot_uncommitted(
2759                    &c.worktree,
2760                    &self
2761                        .state
2762                        .dir()
2763                        .join("artifacts")
2764                        .join(format!("candidate-{label}-recovery")),
2765                )
2766                .await
2767                {
2768                    self.state.event(
2769                        "preserve",
2770                        format!("candidate {label} snapshot incomplete: {e:#}"),
2771                    );
2772                }
2773                let rescued = git::rescue_commit(
2774                    &c.worktree,
2775                    &format!("magi: candidate {label} (uncommitted work)"),
2776                )
2777                .await?;
2778                self.state.note_withheld("implement", &rescued.withheld);
2779                let base = &self.state.base_commit;
2780                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2781                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2782                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2783                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2784                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2785                if c.empty && commits > 0 && !patch.trim().is_empty()
2786                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2787                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2788                        || self.state.tally.is_some())
2789                {
2790                    // Keep the old classification and pending marker so every
2791                    // retry stops too. Retain decisions as recovery evidence;
2792                    // only a supported recovery operation may replace them.
2793                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2794                }
2795                let c = &mut self.state.candidates[i];
2796                c.commits = commits;
2797                c.stat = stat;
2798                c.files = files;
2799                let no_changes = commits == 0 || patch.trim().is_empty();
2800                c.empty = no_changes && !interrupted;
2801                if interrupted && no_changes {
2802                    c.failed = None;
2803                    c.verified_noop = None;
2804                }
2805                if !c.empty {
2806                    c.failed = None;
2807                    c.verified_noop = None;
2808                }
2809                self.state.event(
2810                    "preserve",
2811                    format!("candidate {label}: {files} files, {commits} commits"),
2812                );
2813                Ok::<_, anyhow::Error>(())
2814            }
2815            .await;
2816            match result {
2817                Ok(()) => {
2818                    self.state.candidate_preservation.remove(&label);
2819                    if !interrupted || self.state.candidates[i].files > 0 {
2820                        self.state.interrupted_implementations.remove(&label);
2821                    }
2822                }
2823                Err(e) => {
2824                    let reason = format!("candidate {label} preservation failed: {e:#}");
2825                    self.state.event("preserve", &reason);
2826                    self.state.candidate_preservation.insert(label, reason);
2827                }
2828            }
2829            if self.state.candidate_preservation.is_empty()
2830                && self.state.status == RunStatus::Blocked
2831                && self.state.candidates.iter().all(|c| c.failed.is_none())
2832            {
2833                // Persist the recovered phase with the cleared pending marker;
2834                // a crash between saves must not turn this into a selected-seat
2835                // failure hold on the next resume.
2836                self.state.status = RunStatus::Implementing;
2837                self.state.event(
2838                    "preserve",
2839                    "candidate persistence recovered; implementation can continue",
2840                );
2841            }
2842            self.state.save()?;
2843        }
2844        if !self.state.candidate_preservation.is_empty() {
2845            self.state.status = RunStatus::Blocked;
2846            self.state.save()?;
2847            bail!(
2848                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2849            );
2850        }
2851        Ok(())
2852    }
2853
2854    /// Ask again, once, for work a CLI did and then failed to hand over.
2855    ///
2856    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2857    /// status with an empty response and a usage report showing output tokens,
2858    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2859    /// seven minutes and 14,267 output tokens that arrived as an empty
2860    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2861    ///
2862    /// Two conditions, and both matter:
2863    ///
2864    /// - **Only when the tree is untouched.** Often the agent has already
2865    ///   written its files and only the closing message was lost; the rescue
2866    ///   commit below picks that up and there is nothing to ask for. Re-asking
2867    ///   then would pay for a second implementation of work already on disk.
2868    /// - **Once.** A CLI that drops one stream can drop the next, and this
2869    ///   node is the most expensive in the graph.
2870    ///
2871    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2872    /// dropped reply still carried its `conversation_id`, so the seat is asked
2873    /// to finish what it was doing rather than sent the whole task again. It
2874    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2875    /// node's - for the same reason a re-ranked judge does: restating finished
2876    /// work is not the work.
2877    ///
2878    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2879    /// same way until it resets, while an abandoned conversation is still
2880    /// there to be picked up.
2881    async fn resume_undelivered(
2882        &mut self,
2883        results: &mut [(usize, SeatState, AgentOutcome)],
2884        sent: &[SeatJob],
2885        prompts: &Prompts,
2886        run_id: &str,
2887    ) {
2888        for (wi, seat, out) in results.iter_mut() {
2889            let Some(dropped) = (match &*out {
2890                AgentOutcome::Dropped(o) => o.dropped.clone(),
2891                _ => None,
2892            }) else {
2893                continue;
2894            };
2895            let Some(job) = sent.get(*wi) else { continue };
2896            // Already on disk? Then only the closing message was lost.
2897            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2898                self.state.event(
2899                    "implement",
2900                    format!(
2901                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2902                         work is in the tree",
2903                        seat.key, dropped.output_tokens, dropped.why
2904                    ),
2905                );
2906                continue;
2907            }
2908            // The re-ask only makes sense as a resume: `resume_after_drop`
2909            // says nothing about the task, trusting the seat to still hold it.
2910            // Without a session to resume — sessions disabled, or this CLI's
2911            // drop shape happened not to carry a session id — that prompt
2912            // would open a brand-new conversation with no context at all,
2913            // which is worse than leaving this as the ordinary failure it
2914            // already is.
2915            if !has_context(&job.spec, seat, job.sessions) {
2916                self.state.event(
2917                    "implement",
2918                    format!(
2919                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2920                         is no session left to resume",
2921                        seat.key, dropped.output_tokens, dropped.why
2922                    ),
2923                );
2924                continue;
2925            }
2926            self.state.event(
2927                "implement",
2928                format!(
2929                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2930                     conversation",
2931                    seat.key, dropped.output_tokens, dropped.why
2932                ),
2933            );
2934            let mut retry = job.clone();
2935            retry.seat = seat.clone();
2936            retry.prompt = prompt::resume_after_drop(&dropped.why);
2937            retry.timeout = retry_budget(job.timeout, true);
2938            retry.stem = format!("{}-resume", job.stem);
2939            let cache = self.state.config.cache_dir();
2940            let ctx = WaveCtx {
2941                carry_seats: false,
2942                pauses: Some((&self.pause, &self.interrupt)),
2943                run: run_id,
2944                node: "implement",
2945                prompts,
2946                cache: cache.as_deref(),
2947                round: None,
2948            };
2949            let (resumed_seat, resumed) =
2950                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2951            *seat = resumed_seat;
2952            *out = resumed;
2953        }
2954    }
2955
2956    /// Fall an implement seat through to the next untried agent in the
2957    /// implementer roster when it lost to quota — or, since the handover was
2958    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2959    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2960    /// quota path itself being unchanged — instead of leaving the
2961    /// seat's loss final the moment one agent's account runs dry.
2962    ///
2963    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2964    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2965    /// truncates to the single slot rotation picked, so a solo task whose one
2966    /// implementer hits quota mid-run used to have nothing else to try. This
2967    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2968    /// unrotated roster — which is the only place the *other* candidates in
2969    /// the machine's roster still exist once `implementers` has been cut down
2970    /// to size.
2971    ///
2972    /// Walks forward from just past the seat's own original position in the
2973    /// roster, never wrapping back to the front: a later candidate slot (say
2974    /// `beta`, the roster's second entry) must fall through to the *next*
2975    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2976    /// almost certainly a different candidate's own agent already — and once
2977    /// the roster's tail is exhausted there is nothing left to fall through
2978    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2979    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2980    /// let this retry that id forever. The loop keeps falling through until
2981    /// an attempt lands something other than `Quota` or the roster's tail
2982    /// runs out of untried ids, at which point the seat is left exactly as
2983    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2984    /// `QuotaLoss` recorded, the candidate failed/empty.
2985    ///
2986    /// `sent` is taken mutably and updated with the fallback agent's spec:
2987    /// `resume_unconfirmed_commands`, which runs after this and also reads
2988    /// `sent`, must see whichever agent actually ended up answering the seat
2989    /// — reading the stale, original spec there would check session
2990    /// eligibility against the wrong CLI and could hand a fallback agent's
2991    /// session id to the agent that just lost the seat to quota.
2992    ///
2993    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2994    /// — `self.seat` only reuses state when the agent id is unchanged, so
2995    /// handing it a different id already gets this for free. Reusing the old
2996    /// seat would resume a different CLI's session as if it were a
2997    /// continuation of this one.
2998    ///
2999    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3000    /// a quota loss cuts an agent off mid-turn, so anything already in the
3001    /// tree is unfinished work, not a completed candidate a re-ask would pay
3002    /// for twice. A dirty tree is rescued into a commit first (the same
3003    /// neutral-identity rescue `implement`'s own outcome loop gives every
3004    /// candidate) so the next agent starts clean.
3005    ///
3006    /// The new agent gets the implementer's full prompt and full
3007    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3008    /// it has no session and no context, and is implementing the task from
3009    /// nothing, unlike a resumed drop which is only restating work already
3010    /// done.
3011    ///
3012    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3013    /// `implement` event, never into `self.state.quota` — that is what
3014    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3015    /// go unspent, and a seat that ultimately recovered on its second or
3016    /// third agent is not the stalled panel that check exists to catch. Only
3017    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3018    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3019    /// outcome loop already has — this helper never pushes to it itself.
3020    async fn resume_seat_handovers(
3021        &mut self,
3022        results: &mut [(usize, SeatState, AgentOutcome)],
3023        sent: &mut [SeatJob],
3024        prompts: &Prompts,
3025        run_id: &str,
3026    ) -> Result<()> {
3027        let instruction = seeded_instruction(&self.state);
3028        let language = self.state.config.graph.language.clone();
3029        let github_language = self.state.config.graph.github_language.clone();
3030        let brief = self
3031            .state
3032            .advice
3033            .as_ref()
3034            .and_then(|a| a.synthesis.as_deref())
3035            .map(str::to_owned);
3036        let attachments = self.state.attachments.clone();
3037        for (wi, seat, out) in results.iter_mut() {
3038            // Who holds the other candidate seats of this wave right now
3039            // (earlier handovers already written back to `sent`).
3040            let others: BTreeSet<String> = sent
3041                .iter()
3042                .enumerate()
3043                .filter(|(j, _)| j != wi)
3044                .map(|(_, j)| j.spec.id.clone())
3045                .collect();
3046            let Some(job) = sent.get_mut(*wi) else {
3047                continue;
3048            };
3049            // Where the seat's own original agent sits in the roster — the
3050            // fallback walk starts just past here, never at the front, so a
3051            // later candidate slot's quota loss does not fall back onto an
3052            // earlier slot's own agent.
3053            let start = self
3054                .roles
3055                .implementer_roster
3056                .iter()
3057                .position(|s| s.id == job.spec.id)
3058                .unwrap_or(0);
3059            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3060            let mut fallback_attempt = 0usize;
3061            let mut prev: Option<FailClass> = None;
3062            while let Some(cur) = FailClass::of(&*out) {
3063                if !should_hand_over(prev.as_ref(), &cur) {
3064                    break;
3065                }
3066                let Some(next) =
3067                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3068                        .cloned()
3069                else {
3070                    break;
3071                };
3072                tried.insert(next.id.clone());
3073                fallback_attempt += 1;
3074
3075                let coordination = self.coordinate(&job.cwd).await?;
3076                if let Ok(r) = git::rescue_commit(
3077                    &job.cwd,
3078                    &format!(
3079                        "magi: candidate {} (uncommitted work before {} fallback)",
3080                        seat.key,
3081                        if cur == FailClass::Quota {
3082                            "quota"
3083                        } else {
3084                            "handover"
3085                        }
3086                    ),
3087                )
3088                .await
3089                {
3090                    self.state.note_withheld("implement", &r.withheld);
3091                }
3092                drop(coordination);
3093
3094                record_handover(
3095                    &mut self.state,
3096                    "implement",
3097                    &seat.key,
3098                    &seat.agent,
3099                    &next.id,
3100                    &cur,
3101                    &fail_reason(&*out),
3102                );
3103                prev = Some(cur.clone());
3104
3105                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3106                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3107                // Kept in sync on `sent` itself, not just the local retry: a
3108                // later helper (`resume_unconfirmed_commands`) reads `sent`
3109                // after this one returns and must see whichever agent is now
3110                // occupying the seat, not the one that just quota'd out —
3111                // otherwise it would judge session/continuation eligibility
3112                // by the wrong CLI and could resend a fallback's session id
3113                // to the agent that lost it the seat in the first place.
3114                job.spec = next.clone();
3115                let mut retry = job.clone();
3116                retry.seat = new_seat;
3117                retry.prompt = prompt::implement_in(
3118                    &instruction,
3119                    &job.cwd.to_string_lossy(),
3120                    &language,
3121                    &github_language,
3122                    brief.as_deref(),
3123                    &attachments,
3124                );
3125                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3126                let cache = self.state.config.cache_dir();
3127                let ctx = WaveCtx {
3128                    carry_seats: false,
3129                    pauses: Some((&self.pause, &self.interrupt)),
3130                    run: run_id,
3131                    node: "implement",
3132                    prompts,
3133                    cache: cache.as_deref(),
3134                    round: None,
3135                };
3136                let (fallback_seat, fallback_out) = run_one(
3137                    retry,
3138                    Arc::clone(&self.sem),
3139                    &ctx,
3140                    &mut self.state,
3141                    fallback_attempt,
3142                )
3143                .await;
3144                *seat = fallback_seat;
3145                *out = fallback_out;
3146            }
3147        }
3148        Ok(())
3149    }
3150
3151    /// Ask an implement seat's own CLI to confirm what it started, once, when
3152    /// its reply reported a command whose completion status it never
3153    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3154    /// that does and does not mean.
3155    ///
3156    /// The completion contract this task asks for, extended to `implement`
3157    /// with the same signal `continue_fix_report` reads for the fixer,
3158    /// rather than a keyword search over the reply or a hard requirement on
3159    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3160    /// a candidate's CLI turn ended cleanly while a test run it had started
3161    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3162    /// named in it at all is untouched by this: `commands` is empty, so
3163    /// there is nothing to be unconfirmed.
3164    ///
3165    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3166    /// this is not about recovering edits that might already be on disk, it
3167    /// is about a result the seat itself never vouched for, which resuming
3168    /// asks for regardless of what the tree already holds. Bounded to one
3169    /// attempt for the same reason `resume_undelivered` is — this is the
3170    /// most expensive node in the graph — and a seat that still cannot
3171    /// confirm on that attempt is left as whatever its (possibly still
3172    /// unconfirmed) reply says; this does not invent a new "failed" reason
3173    /// for a candidate that otherwise produced a real, committed change.
3174    async fn resume_unconfirmed_commands(
3175        &mut self,
3176        results: &mut [(usize, SeatState, AgentOutcome)],
3177        sent: &[SeatJob],
3178        prompts: &Prompts,
3179        run_id: &str,
3180    ) {
3181        for (wi, seat, out) in results.iter_mut() {
3182            let AgentOutcome::Ok(o) = &*out else {
3183                continue;
3184            };
3185            if !has_unconfirmed_command(&o.commands) {
3186                continue;
3187            }
3188            let Some(job) = sent.get(*wi) else { continue };
3189            if !has_context(&job.spec, seat, job.sessions) {
3190                self.state.event(
3191                    "implement",
3192                    format!(
3193                        "{}: the reply named a command whose own CLI never confirmed the exit \
3194                         status of, but there is no session left to resume",
3195                        seat.key
3196                    ),
3197                );
3198                continue;
3199            }
3200            self.state.event(
3201                "implement",
3202                format!(
3203                    "{}: the reply named a command whose own CLI never confirmed the exit \
3204                     status of; resuming the conversation",
3205                    seat.key
3206                ),
3207            );
3208            let mut retry = job.clone();
3209            retry.seat = seat.clone();
3210            retry.prompt = prompt::resume_incomplete(
3211                "a command in your last reply had no confirmed exit status",
3212            );
3213            retry.timeout = retry_budget(job.timeout, true);
3214            retry.stem = format!("{}-confirm", job.stem);
3215            let cache = self.state.config.cache_dir();
3216            let ctx = WaveCtx {
3217                carry_seats: false,
3218                pauses: Some((&self.pause, &self.interrupt)),
3219                run: run_id,
3220                node: "implement",
3221                prompts,
3222                cache: cache.as_deref(),
3223                round: None,
3224            };
3225            let (resumed_seat, resumed) =
3226                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3227            *seat = resumed_seat;
3228            *out = resumed;
3229        }
3230    }
3231
3232    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3233    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3234    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3235    /// that motivated this.
3236    ///
3237    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3238    /// own nudge loop already covers for judge/review/vote seats, and not a
3239    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3240    /// implement seats: here the CLI turn genuinely finished while the node's
3241    /// own work — the fixer's account of what it did — had not. Gated purely
3242    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3243    /// reply, never on any wording in it, so a fixer whose valid, first-try
3244    /// `FixReport` happens to mention having waited on a background test is
3245    /// never resumed — the `Ok(report)` branch at the call site returns
3246    /// before this is ever invoked.
3247    ///
3248    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3249    /// attempt ([`retry_budget`]), nothing attempted once the session is
3250    /// gone, and a quota hit ends the loop immediately rather than retrying a
3251    /// rate limit that fails the same way again.
3252    async fn continue_fix_report(
3253        &mut self,
3254        mut seat: SeatState,
3255        parse_err: String,
3256        job: &SeatJob,
3257        prompts: &Prompts,
3258        run_id: &str,
3259        round: usize,
3260    ) -> (
3261        SeatState,
3262        Option<FixReport>,
3263        Option<String>,
3264        ContinuationRecord,
3265    ) {
3266        let mut last_err = parse_err;
3267        let mut cumulative_wait_ms = 0u64;
3268        let mut attempts = 0usize;
3269        loop {
3270            if !has_context(&job.spec, &seat, job.sessions) {
3271                self.state.event(
3272                    "fix",
3273                    format!(
3274                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3275                         session left to resume into"
3276                    ),
3277                );
3278                let outcome = if attempts == 0 {
3279                    ContinuationOutcome::NoSession
3280                } else {
3281                    ContinuationOutcome::Exhausted
3282                };
3283                return (
3284                    seat,
3285                    None,
3286                    Some(format!("unparsable fix report: {last_err}")),
3287                    ContinuationRecord {
3288                        attempts,
3289                        cumulative_wait_ms,
3290                        outcome,
3291                    },
3292                );
3293            }
3294            if attempts >= MAX_FIX_CONTINUATIONS {
3295                self.state.event(
3296                    "fix",
3297                    format!(
3298                        "round {round}: fixer's reply still had no adoption report after \
3299                         {attempts} continuation(s) ({last_err}); giving up"
3300                    ),
3301                );
3302                return (
3303                    seat,
3304                    None,
3305                    Some(format!(
3306                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3307                    )),
3308                    ContinuationRecord {
3309                        attempts,
3310                        cumulative_wait_ms,
3311                        outcome: ContinuationOutcome::Exhausted,
3312                    },
3313                );
3314            }
3315            attempts += 1;
3316            self.state.event(
3317                "fix",
3318                format!(
3319                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3320                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3321                ),
3322            );
3323            let mut retry = job.clone();
3324            retry.seat = seat.clone();
3325            retry.prompt = prompt::resume_incomplete(&last_err);
3326            retry.timeout = retry_budget(job.timeout, true);
3327            retry.stem = format!("{}-continue{attempts}", job.stem);
3328            let cache = self.state.config.cache_dir();
3329            let ctx = WaveCtx {
3330                carry_seats: false,
3331                pauses: Some((&self.pause, &self.interrupt)),
3332                run: run_id,
3333                node: "fix",
3334                prompts,
3335                cache: cache.as_deref(),
3336                round: Some(round),
3337            };
3338            let (resumed_seat, resumed_out) = run_one(
3339                retry,
3340                Arc::clone(&self.sem),
3341                &ctx,
3342                &mut self.state,
3343                attempts,
3344            )
3345            .await;
3346            seat = resumed_seat;
3347            match resumed_out {
3348                AgentOutcome::Ok(o) => {
3349                    cumulative_wait_ms += o.duration_ms;
3350                    match verdict::extract_json::<FixReport>(&o.text) {
3351                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3352                            self.state.event(
3353                                "fix",
3354                                format!(
3355                                    "round {round}: fixer's adoption report recovered after \
3356                                     {attempts} continuation(s)"
3357                                ),
3358                            );
3359                            return (
3360                                seat,
3361                                Some(report),
3362                                None,
3363                                ContinuationRecord {
3364                                    attempts,
3365                                    cumulative_wait_ms,
3366                                    outcome: ContinuationOutcome::Resumed,
3367                                },
3368                            );
3369                        }
3370                        // The report parsed, but this same reply's own
3371                        // CommandEvidence — the identical record `state.jobs`
3372                        // renders — names a command whose CLI never
3373                        // confirmed an exit status. Read together, that is
3374                        // not a resolved answer: keep nudging rather than
3375                        // accept a report standing next to a command the
3376                        // seat's own CLI cannot vouch for.
3377                        Ok(_) => {
3378                            last_err = "the reply parsed, but it reported a command whose own CLI \
3379                                 never confirmed an exit status"
3380                                .to_owned();
3381                        }
3382                        Err(e) => last_err = e.to_string(),
3383                    }
3384                }
3385                AgentOutcome::Quota(o) => {
3386                    cumulative_wait_ms += o.duration_ms;
3387                    self.state.quota.push(QuotaLoss {
3388                        seat: seat.key.clone(),
3389                        node: "fix".to_owned(),
3390                        at: Timestamp::now(),
3391                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3392                    });
3393                    self.state.event(
3394                        "fix",
3395                        format!(
3396                            "round {round}: continuation rate limited (quota); not retrying now"
3397                        ),
3398                    );
3399                    return (
3400                        seat,
3401                        None,
3402                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3403                        ContinuationRecord {
3404                            attempts,
3405                            cumulative_wait_ms,
3406                            outcome: ContinuationOutcome::QuotaLost,
3407                        },
3408                    );
3409                }
3410                AgentOutcome::Dropped(o) => {
3411                    cumulative_wait_ms += o.duration_ms;
3412                    let why = o
3413                        .dropped
3414                        .as_ref()
3415                        .map(|d| d.why.as_str())
3416                        .unwrap_or("the CLI ended the stream without delivering its answer");
3417                    last_err = format!("the CLI dropped the stream ({why})");
3418                }
3419                AgentOutcome::Failed(e) => last_err = e,
3420                // The continuation was never started; the report stays lost
3421                // and the caller records that, without any handover.
3422                AgentOutcome::ResourceBlocked(e) => {
3423                    return (
3424                        seat,
3425                        None,
3426                        Some(format!("the continuation was not started: {e}")),
3427                        ContinuationRecord {
3428                            attempts: attempts.saturating_sub(1),
3429                            cumulative_wait_ms,
3430                            outcome: ContinuationOutcome::NoSession,
3431                        },
3432                    );
3433                }
3434            }
3435        }
3436    }
3437
3438    fn after_implement(&mut self) -> Result<()> {
3439        if !self.state.candidate_preservation.is_empty() {
3440            bail!("candidate preservation incomplete; judging is unsafe");
3441        }
3442        // Scan every candidate patch once the set is complete.
3443        if self.state.leaks.is_empty() {
3444            let cfg = self.state.config.blind.clone();
3445            let mut leaks = Vec::new();
3446            for c in &self.state.candidates {
3447                let Some(patch) =
3448                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3449                else {
3450                    continue;
3451                };
3452                leaks.extend(blind::scan(
3453                    &format!("candidate {} patch", c.label),
3454                    &patch,
3455                    &cfg.vendor_tokens,
3456                ));
3457            }
3458            if !leaks.is_empty() {
3459                let summary = leaks
3460                    .iter()
3461                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3462                    .collect::<Vec<_>>()
3463                    .join(", ");
3464                match cfg.on_leak {
3465                    LeakPolicy::Fail => {
3466                        self.state.status = RunStatus::Failed;
3467                        self.state
3468                            .event("blind", format!("vendor text in a patch: {summary}"));
3469                        self.state.leaks = leaks;
3470                        self.state.save()?;
3471                        self.settle_questions();
3472                        bail!(
3473                            "blind.on_leak = \"fail\" and vendor text reached a \
3474                             judged patch: {summary}"
3475                        );
3476                    }
3477                    LeakPolicy::Redact => self.state.event(
3478                        "blind",
3479                        format!("redacting vendor text for judging: {summary}"),
3480                    ),
3481                    LeakPolicy::Warn => self.state.event(
3482                        "blind",
3483                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3484                    ),
3485                }
3486                self.state.leaks = leaks;
3487            }
3488        }
3489
3490        if self.state.viable().is_empty() {
3491            if self.state.all_candidates_verified_noop() {
3492                // Every candidate agreed, with evidence the adoption guard
3493                // accepted, that nothing belongs in this worktree. That is
3494                // not the same fact as a candidate that simply failed to
3495                // write anything, and settling it as an ordinary `Failed`
3496                // (see `SCHEMA`'s doc for schema 10) is what let two of
3497                // task 391f's attempts burn a retry each re-discovering the
3498                // same already-landed fix. Terminal either way, so `judge`
3499                // must never run over an empty candidate set — unlike the
3500                // `Failed` branch below this returns `Ok`, not an error:
3501                // nothing here failed.
3502                self.state.status = RunStatus::VerifiedNoop;
3503                self.state.save()?;
3504                self.settle_questions();
3505                return Ok(());
3506            }
3507            self.state.status = RunStatus::Failed;
3508            self.state.save()?;
3509            self.settle_questions();
3510            bail!("no candidate produced a change; nothing to judge");
3511        }
3512        self.state.status = RunStatus::Judging;
3513        self.state.save()?;
3514        Ok(())
3515    }
3516
3517    // --------------------------------------------------------------- judge
3518
3519    async fn judge(&mut self) -> Result<()> {
3520        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3521        // agent files with `magi task add` name the run that paid for it. The
3522        // prompt overlay is cloned alongside it because the waves borrow it
3523        // while `self` is mutably borrowed by the node's own bookkeeping.
3524        let run_id = self.state.id.clone();
3525        let prompts = self.state.config.prompts.clone();
3526        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3527            return Ok(());
3528        }
3529        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3530        if viable.len() == 1 {
3531            // Recorded so this is a one-time event: `judgements` stays empty
3532            // either way, which without this flag is indistinguishable from
3533            // "not yet judged" on the next reentry — and status is left
3534            // untouched, so a later node's conclusion (e.g. `Blocked` after
3535            // the review budget ran out) survives a resume instead of being
3536            // clobbered back to `Judging` by this node running again.
3537            self.state.judge_skipped = true;
3538            self.state.event(
3539                "judge",
3540                format!(
3541                    "only candidate {} produced a change; judging skipped",
3542                    viable[0].label
3543                ),
3544            );
3545            self.state.save()?;
3546            return Ok(());
3547        }
3548        self.state.status = RunStatus::Judging;
3549
3550        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3551        let language = self.state.config.graph.language.clone();
3552        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3553        let sessions = self.state.config.graph.sessions;
3554        let artifacts = agent::artifacts_dir(&self.state.dir());
3555        let root = self.state.worktree_root();
3556        let base_short = short(&self.state.base_commit);
3557
3558        let mut jobs = Vec::new();
3559        let mut orders = Vec::new();
3560        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3561            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3562            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3563            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3564            let seat_key = format!("judge-{}", j + 1);
3565            let seat = self.seat(&seat_key, &spec.id);
3566            jobs.push(SeatJob {
3567                prompt: prompt::judge(
3568                    &self.state.instruction,
3569                    &views,
3570                    self.roles.judges.len(),
3571                    &base_short,
3572                    &language,
3573                ),
3574                spec,
3575                seat,
3576                cwd: root.join(format!("judge-{}", j + 1)),
3577                timeout,
3578                allow_write: false,
3579                sessions,
3580                artifacts: artifacts.clone(),
3581                stem: format!("judge-{}", j + 1),
3582                handover: None,
3583            });
3584        }
3585
3586        self.state.event(
3587            "judge",
3588            format!(
3589                "{} judges ranking {} candidates blind",
3590                jobs.len(),
3591                viable.len()
3592            ),
3593        );
3594        let labels_for_check = labels.clone();
3595        let mut quota_losses = Vec::new();
3596        let cache = self.state.config.cache_dir();
3597        let ctx = WaveCtx {
3598            carry_seats: false,
3599            pauses: Some((&self.pause, &self.interrupt)),
3600            run: &run_id,
3601            node: "judge",
3602            prompts: &prompts,
3603            cache: cache.as_deref(),
3604            round: None,
3605        };
3606        let results = ask_json_wave::<Ranking>(
3607            jobs,
3608            Arc::clone(&self.sem),
3609            self.state.config.graph.retries,
3610            &self.roles.judge_roster,
3611            &ctx,
3612            &mut quota_losses,
3613            &mut self.state,
3614            &move |r: &Ranking| r.validate(&labels_for_check),
3615        )
3616        .await;
3617        if self.state.resource_block.is_some() {
3618            return self.stop_for_resource();
3619        }
3620        self.state.quota.extend(quota_losses);
3621
3622        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3623            let agent_id = seat.agent.clone();
3624            self.state.seats.insert(seat.key.clone(), seat);
3625            let mut record = Judgement {
3626                judge: j + 1,
3627                seat: format!("judge-{}", j + 1),
3628                agent: agent_id,
3629                ranking: Vec::new(),
3630                reasons: BTreeMap::new(),
3631                confidence: None,
3632                strengths: BTreeMap::new(),
3633                order: orders[j].clone(),
3634                failed: None,
3635                duration_ms: 0,
3636            };
3637            match res {
3638                Ok((ranking, out)) => {
3639                    record.ranking = ranking.normalized();
3640                    record.reasons = ranking.reasons;
3641                    record.strengths = ranking.strengths;
3642                    record.confidence = ranking.confidence;
3643                    record.duration_ms = out.duration_ms;
3644                    self.state.event(
3645                        "judge",
3646                        format!(
3647                            "judge {} ranked {}",
3648                            j + 1,
3649                            record.ranking.iter().collect::<String>()
3650                        ),
3651                    );
3652                }
3653                Err(e) => {
3654                    record.failed = Some(e.to_string());
3655                    self.state
3656                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3657                }
3658            }
3659            self.state.judgements.push(record);
3660            self.state.save()?;
3661        }
3662        Ok(())
3663    }
3664
3665    // ---------------------------------------------------------- deliberate
3666
3667    async fn deliberate(&mut self) -> Result<()> {
3668        if self.hold_base_sync().await? {
3669            return Ok(());
3670        }
3671        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3672        // agent files with `magi task add` name the run that paid for it. The
3673        // prompt overlay is cloned alongside it because the waves borrow it
3674        // while `self` is mutably borrowed by the node's own bookkeeping.
3675        let run_id = self.state.id.clone();
3676        let prompts = self.state.config.prompts.clone();
3677        if !self.state.deliberation.is_empty() || self.state.tally.is_some() {
3678            return Ok(());
3679        }
3680        let tops: Vec<char> = self
3681            .state
3682            .judgements
3683            .iter()
3684            .filter_map(|j| j.ranking.first().copied())
3685            .collect();
3686        let rounds = self.state.config.graph.deliberate_rounds;
3687        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3688            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3689                self.state.event(
3690                    "deliberate",
3691                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3692                );
3693            }
3694            self.state.status = RunStatus::Voting;
3695            self.state.save()?;
3696            return Ok(());
3697        }
3698
3699        self.state.status = RunStatus::Deliberating;
3700        self.state.event(
3701            "deliberate",
3702            format!(
3703                "split: first choices were {} — opening {rounds} round(s)",
3704                tops.iter().collect::<String>()
3705            ),
3706        );
3707
3708        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3709        let language = self.state.config.graph.language.clone();
3710        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3711        let sessions = self.state.config.graph.sessions;
3712        let artifacts = agent::artifacts_dir(&self.state.dir());
3713        let root = self.state.worktree_root();
3714        let base_short = short(&self.state.base_commit);
3715
3716        // Judges argue in sequence so that a turn can answer the one before it;
3717        // that is the difference between deliberation and three parallel
3718        // monologues.
3719        for round in 1..=rounds {
3720            let mut turns: Vec<DeliberationTurn> = Vec::new();
3721            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3722                if self.state.judgements[j].failed.is_some() {
3723                    continue;
3724                }
3725                let seat_key = format!("judge-{}", j + 1);
3726                let spec = self.occupant(&seat_key, spec);
3727                let mut seat = self.seat(&seat_key, &spec.id);
3728                let transcript = self.transcript(&turns, j);
3729                let build = |context: Option<&str>| {
3730                    prompt::deliberate(
3731                        &self.state.instruction,
3732                        context,
3733                        &transcript,
3734                        round,
3735                        rounds,
3736                        &language,
3737                    )
3738                };
3739                let block = self.candidate_block(&viable, &base_short);
3740                let full = build(Some(&block));
3741                let text = if has_context(&spec, &seat, sessions) {
3742                    build(None)
3743                } else {
3744                    full.clone()
3745                };
3746                let job = SeatJob {
3747                    spec,
3748                    seat: seat.clone(),
3749                    prompt: text,
3750                    cwd: root.join(format!("judge-{}", j + 1)),
3751                    timeout,
3752                    allow_write: false,
3753                    sessions,
3754                    artifacts: artifacts.clone(),
3755                    stem: format!("delib-{round}-judge-{}", j + 1),
3756                    handover: Some(full),
3757                };
3758                let cache = self.state.config.cache_dir();
3759                let ctx = WaveCtx {
3760                    carry_seats: false,
3761                    pauses: Some((&self.pause, &self.interrupt)),
3762                    run: &run_id,
3763                    node: "deliberate",
3764                    prompts: &prompts,
3765                    cache: cache.as_deref(),
3766                    round: None,
3767                };
3768                // A turn is never nudged (`retries` 0): a failed seat is
3769                // handed to the next roster agent, which gets the full
3770                // context. An empty answer is a turn, not a failure.
3771                let mut losses = Vec::new();
3772                let mut results = ask_wave_with::<String>(
3773                    vec![job],
3774                    Arc::clone(&self.sem),
3775                    0,
3776                    &self.roles.judge_roster,
3777                    &ctx,
3778                    &mut losses,
3779                    &mut self.state,
3780                    &|text: &str| {
3781                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3782                    },
3783                )
3784                .await;
3785                self.state.quota.extend(losses);
3786                let (updated, res, _) = results.pop().expect("one job in, one result out");
3787                seat = updated;
3788                let agent_id = seat.agent.clone();
3789                self.state.seats.insert(seat.key.clone(), seat);
3790                let body = match res {
3791                    Ok((body, _)) => body,
3792                    // Skip the seat; a CLI's raw error JSON is never read as
3793                    // this judge's position.
3794                    Err(e) => {
3795                        self.state
3796                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3797                        continue;
3798                    }
3799                };
3800                let tentative = verdict::extract_json::<Position>(&body)
3801                    .ok()
3802                    .and_then(|p| p.tentative)
3803                    .and_then(|s| s.trim().chars().next())
3804                    .map(|c| c.to_ascii_uppercase());
3805                self.state.event(
3806                    "deliberate",
3807                    format!(
3808                        "round {round}: judge {} now favours {}",
3809                        j + 1,
3810                        tentative.map_or("—".to_owned(), |c| c.to_string())
3811                    ),
3812                );
3813                turns.push(DeliberationTurn {
3814                    judge: j + 1,
3815                    agent: agent_id,
3816                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3817                    tentative,
3818                });
3819            }
3820            self.state
3821                .deliberation
3822                .push(DeliberationRound { round, turns });
3823            self.state.save()?;
3824        }
3825
3826        self.state.status = RunStatus::Voting;
3827        self.state.save()?;
3828        Ok(())
3829    }
3830
3831    // ---------------------------------------------------------------- vote
3832
3833    async fn vote(&mut self) -> Result<()> {
3834        if self.hold_base_sync().await? {
3835            return Ok(());
3836        }
3837        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3838        // agent files with `magi task add` name the run that paid for it. The
3839        // prompt overlay is cloned alongside it because the waves borrow it
3840        // while `self` is mutably borrowed by the node's own bookkeeping.
3841        let run_id = self.state.id.clone();
3842        let prompts = self.state.config.prompts.clone();
3843        if !self.state.votes.is_empty() || self.state.tally.is_some() {
3844            return Ok(());
3845        }
3846        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3847        if viable.len() == 1 {
3848            return Ok(());
3849        }
3850        self.state.status = RunStatus::Voting;
3851
3852        let language = self.state.config.graph.language.clone();
3853        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3854        let sessions = self.state.config.graph.sessions;
3855        let artifacts = agent::artifacts_dir(&self.state.dir());
3856        let root = self.state.worktree_root();
3857        let base_short = short(&self.state.base_commit);
3858        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3859
3860        let mut jobs = Vec::new();
3861        let mut seats_at = Vec::new();
3862        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3863            if self
3864                .state
3865                .judgements
3866                .get(j)
3867                .is_some_and(|r| r.failed.is_some())
3868            {
3869                continue;
3870            }
3871            let seat_key = format!("judge-{}", j + 1);
3872            let spec = self.occupant(&seat_key, spec);
3873            let seat = self.seat(&seat_key, &spec.id);
3874            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3875            let text = if has_context(&spec, &seat, sessions) {
3876                prompt::final_vote(&viable, &language)
3877            } else {
3878                full.clone()
3879            };
3880            jobs.push(SeatJob {
3881                spec,
3882                seat,
3883                prompt: text,
3884                cwd: root.join(format!("judge-{}", j + 1)),
3885                timeout,
3886                allow_write: false,
3887                sessions,
3888                artifacts: artifacts.clone(),
3889                stem: format!("vote-judge-{}", j + 1),
3890                handover: Some(full),
3891            });
3892            seats_at.push(j);
3893        }
3894
3895        self.state.event(
3896            "vote",
3897            format!(
3898                "collecting {} final votes one by one, privately",
3899                jobs.len()
3900            ),
3901        );
3902        let allowed = viable.clone();
3903        let mut quota_losses = Vec::new();
3904        let cache = self.state.config.cache_dir();
3905        let ctx = WaveCtx {
3906            carry_seats: false,
3907            pauses: Some((&self.pause, &self.interrupt)),
3908            run: &run_id,
3909            node: "vote",
3910            prompts: &prompts,
3911            cache: cache.as_deref(),
3912            round: None,
3913        };
3914        let results = ask_json_wave::<FinalVote>(
3915            jobs,
3916            Arc::clone(&self.sem),
3917            self.state.config.graph.retries,
3918            &self.roles.judge_roster,
3919            &ctx,
3920            &mut quota_losses,
3921            &mut self.state,
3922            &move |v: &FinalVote| match v.label() {
3923                Some(c) if allowed.contains(&c) => Ok(()),
3924                other => bail!("vote {other:?} is not one of {allowed:?}"),
3925            },
3926        )
3927        .await;
3928        if self.state.resource_block.is_some() {
3929            return self.stop_for_resource();
3930        }
3931        self.state.quota.extend(quota_losses);
3932
3933        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3934            let agent_id = seat.agent.clone();
3935            self.state.seats.insert(seat.key.clone(), seat);
3936            let initial = self
3937                .state
3938                .judgements
3939                .get(j)
3940                .and_then(|r| r.ranking.first().copied());
3941            let mut record = VoteRecord {
3942                judge: j + 1,
3943                agent: agent_id,
3944                vote: None,
3945                reason: String::new(),
3946                changed: false,
3947            };
3948            match res {
3949                Ok((v, _)) => {
3950                    record.vote = v.label();
3951                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3952                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3953                    self.state.event(
3954                        "vote",
3955                        format!(
3956                            "judge {} voted {}{}",
3957                            j + 1,
3958                            record.vote.unwrap_or('?'),
3959                            if record.changed { " (changed)" } else { "" }
3960                        ),
3961                    );
3962                }
3963                Err(e) => {
3964                    self.state
3965                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3966                }
3967            }
3968            self.state.votes.push(record);
3969            self.state.save()?;
3970        }
3971        Ok(())
3972    }
3973
3974    // --------------------------------------------------------------- tally
3975
3976    fn tally(&mut self) -> Result<()> {
3977        if self.state.tally.is_some() {
3978            return Ok(());
3979        }
3980        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3981        let tops: Vec<char> = self
3982            .state
3983            .judgements
3984            .iter()
3985            .filter_map(|j| j.ranking.first().copied())
3986            .collect();
3987        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3988
3989        // A judge whose private vote failed still counted once, in the initial
3990        // ranking; using it beats discarding a whole seat.
3991        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3992        let mut cast: Vec<char> = Vec::new();
3993        for (i, j) in self.state.judgements.iter().enumerate() {
3994            let vote = self
3995                .state
3996                .votes
3997                .iter()
3998                .find(|v| v.judge == i + 1)
3999                .and_then(|v| v.vote)
4000                .or_else(|| j.ranking.first().copied());
4001            if let Some(v) = vote {
4002                *first_choice.entry(v).or_insert(0) += 1;
4003                cast.push(v);
4004            }
4005        }
4006
4007        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4008        for j in &self.state.judgements {
4009            let n = j.ranking.len();
4010            for (pos, label) in j.ranking.iter().enumerate() {
4011                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4012            }
4013        }
4014
4015        let best = first_choice.values().copied().max().unwrap_or(0);
4016        let mut leaders: Vec<char> = first_choice
4017            .iter()
4018            .filter(|(_, v)| **v == best)
4019            .map(|(k, _)| *k)
4020            .collect();
4021        let mut tie_break = None;
4022        if leaders.len() > 1 {
4023            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4024            let borda_leaders: Vec<char> = leaders
4025                .iter()
4026                .copied()
4027                .filter(|l| borda[l] == top_borda)
4028                .collect();
4029            tie_break = Some(if borda_leaders.len() == 1 {
4030                format!(
4031                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4032                    leaders.len()
4033                )
4034            } else {
4035                format!(
4036                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4037                    leaders.len()
4038                )
4039            });
4040            leaders = borda_leaders;
4041            leaders.sort_unstable();
4042        }
4043        let winner = *leaders
4044            .first()
4045            .or(viable.first())
4046            .context("no candidate to declare a winner from")?;
4047
4048        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4049        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4050        let deliberated = !self.state.deliberation.is_empty();
4051
4052        // Whose verdict is this? A rate-limited seat is absent even if it
4053        // ranked before the limit hit, so presence is measured against the
4054        // recorded losses, not just "did a ranking ever appear".
4055        let quota_seats: std::collections::BTreeSet<&str> =
4056            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4057        let mut present = 0usize;
4058        for (i, j) in self.state.judgements.iter().enumerate() {
4059            if quota_seats.contains(j.seat.as_str()) {
4060                continue;
4061            }
4062            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4063            let voted = self
4064                .state
4065                .votes
4066                .iter()
4067                .any(|v| v.judge == i + 1 && v.vote.is_some());
4068            if ranked || voted {
4069                present += 1;
4070            }
4071        }
4072        // Strict majority of the configured panel. A bare majority is real
4073        // signal we can act on, while a minority verdict must never stand in
4074        // for a healthy one. A one-candidate run needs no panel at all, and
4075        // `judges` stays `0` rather than the roster size a panel that never
4076        // sat would otherwise be credited with.
4077        let needs_quorum = viable.len() > 1;
4078        let judges_total = if needs_quorum {
4079            self.roles.judges.len()
4080        } else {
4081            0
4082        };
4083        let quorum = if needs_quorum {
4084            judges_total / 2 + 1
4085        } else {
4086            0
4087        };
4088        let met_quorum = !needs_quorum || present >= quorum;
4089        let uncontested = (!needs_quorum).then(|| {
4090            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4091        });
4092
4093        self.state.event(
4094            "tally",
4095            match &uncontested {
4096                Some(reason) => format!("winner {winner} — {reason}"),
4097                None => format!(
4098                    "winner {winner} — votes {} | initial {} | {} changed | \
4099                     {present}/{judges_total} judges{}",
4100                    first_choice
4101                        .iter()
4102                        .map(|(k, v)| format!("{k}:{v}"))
4103                        .collect::<Vec<_>>()
4104                        .join(" "),
4105                    if unanimous_initial {
4106                        "unanimous"
4107                    } else {
4108                        "split"
4109                    },
4110                    changed_votes,
4111                    if met_quorum {
4112                        String::new()
4113                    } else {
4114                        format!(" — below quorum ({quorum} required)")
4115                    },
4116                ),
4117            },
4118        );
4119        if !met_quorum {
4120            self.state.event(
4121                "stall",
4122                format!(
4123                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4124                     the run stops here, resumable"
4125                ),
4126            );
4127        }
4128        self.state.tally = Some(Tally {
4129            first_choice,
4130            borda,
4131            winner,
4132            rankings: tops.len(),
4133            unanimous_initial,
4134            deliberated,
4135            changed_votes,
4136            unanimous_final,
4137            tie_break,
4138            judges: judges_total,
4139            present,
4140            quorum,
4141            met_quorum,
4142            uncontested,
4143        });
4144        self.state.status = if met_quorum {
4145            RunStatus::Reviewing
4146        } else {
4147            RunStatus::Stalled
4148        };
4149        self.state.save()?;
4150        Ok(())
4151    }
4152
4153    // ------------------------------------------------------------- recover
4154
4155    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4156    /// resumed toward completion once the transient cause clears.
4157    ///
4158    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4159    /// it toward the quorum, which is exactly the set of seats whose absence
4160    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4161    /// not depend on which node happened to hit the limit), or an ordinary
4162    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4163    /// seat is never disturbed.
4164    ///
4165    /// A seat that now answers with a usable ranking is "recovered": its
4166    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4167    /// `tally` counts it present again), and its vote re-collected. A seat that
4168    /// still fails keeps its loss and stays absent.
4169    ///
4170    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4171    /// to review/gate/merge), `false` when it is still below quorum (the run
4172    /// stays `Stalled`, still resumable for a later retry).
4173    #[allow(clippy::too_many_lines)]
4174    async fn recover_stall(&mut self) -> Result<bool> {
4175        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4176        // agent files with `magi task add` name the run that paid for it. The
4177        // prompt overlay is cloned alongside it because the waves borrow it
4178        // while `self` is mutably borrowed by the node's own bookkeeping.
4179        let run_id = self.state.id.clone();
4180        let prompts = self.state.config.prompts.clone();
4181        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4182        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4183        // is what keeps a non-quota collapse — or a quota loss recorded at the
4184        // deliberate node — from being a permanent dead-end on `--resume`.
4185        let quota_seats: BTreeSet<&str> =
4186            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4187        let absent: Vec<String> = self
4188            .state
4189            .judgements
4190            .iter()
4191            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4192            .map(|j| j.seat.clone())
4193            .collect();
4194        if absent.is_empty() {
4195            return Ok(false);
4196        }
4197        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4198        if viable.len() <= 1 {
4199            return Ok(false);
4200        }
4201        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4202        let language = self.state.config.graph.language.clone();
4203        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4204        let sessions = self.state.config.graph.sessions;
4205        let artifacts = agent::artifacts_dir(&self.state.dir());
4206        let root = self.state.worktree_root();
4207        let base_short = short(&self.state.base_commit);
4208        let candidates: Vec<Candidate> = viable.clone();
4209
4210        // Map each absent seat key to its 0-based position in `roles.judges`.
4211        let mut positions: Vec<usize> = absent
4212            .iter()
4213            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4214            .collect();
4215        if positions.is_empty() {
4216            return Ok(false);
4217        }
4218        positions.sort_unstable();
4219        positions.dedup();
4220
4221        // Re-rank the lost seats, one blind prompt each.
4222        let mut judge_jobs = Vec::new();
4223        for &j in &positions {
4224            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4225            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4226            let seat_key = format!("judge-{}", j + 1);
4227            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4228            let seat = self.seat(&seat_key, &spec.id);
4229            judge_jobs.push(SeatJob {
4230                spec,
4231                seat,
4232                prompt: prompt::judge(
4233                    &self.state.instruction,
4234                    &views,
4235                    self.roles.judges.len(),
4236                    &base_short,
4237                    &language,
4238                ),
4239                cwd: root.join(seat_key),
4240                timeout,
4241                allow_write: false,
4242                sessions,
4243                artifacts: artifacts.clone(),
4244                stem: format!("judge-{}-recover", j + 1),
4245                handover: None,
4246            });
4247        }
4248
4249        let labels_for_check = labels.clone();
4250        let mut judge_losses = Vec::new();
4251        let retries = self.state.config.graph.retries;
4252        let cache = self.state.config.cache_dir();
4253        let ctx = WaveCtx {
4254            carry_seats: false,
4255            pauses: Some((&self.pause, &self.interrupt)),
4256            run: &run_id,
4257            node: "judge",
4258            prompts: &prompts,
4259            cache: cache.as_deref(),
4260            round: None,
4261        };
4262        let results = ask_json_wave::<Ranking>(
4263            judge_jobs,
4264            Arc::clone(&self.sem),
4265            retries,
4266            &self.roles.judge_roster,
4267            &ctx,
4268            &mut judge_losses,
4269            &mut self.state,
4270            &move |r: &Ranking| r.validate(&labels_for_check),
4271        )
4272        .await;
4273        if self.state.resource_block.is_some() {
4274            self.stop_for_resource()?;
4275            return Ok(false);
4276        }
4277
4278        // Refresh the judgement of every seat that ranked again.
4279        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4280        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4281            let agent_id = seat.agent.clone();
4282            self.state.seats.insert(seat.key.clone(), seat);
4283            let record = &mut self.state.judgements[j];
4284            match res {
4285                Ok((ranking, out)) => {
4286                    record.agent = agent_id;
4287                    record.ranking = ranking.normalized();
4288                    record.reasons = ranking.reasons;
4289                    record.strengths = ranking.strengths;
4290                    record.confidence = ranking.confidence;
4291                    record.failed = None;
4292                    record.duration_ms = out.duration_ms;
4293                    recovered.insert(j);
4294                    self.state.event(
4295                        "recover",
4296                        format!("judge {} ranked again after the limit", j + 1),
4297                    );
4298                }
4299                Err(e) => {
4300                    self.state
4301                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4302                }
4303            }
4304        }
4305
4306        // Re-ask the votes of the seats that recovered a ranking.
4307        let mut vote_jobs = Vec::new();
4308        let mut vote_pos: Vec<usize> = Vec::new();
4309        for &j in &recovered {
4310            let seat_key = format!("judge-{}", j + 1);
4311            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4312            let seat = self.seat(&seat_key, &spec.id);
4313            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4314            let text = if has_context(&spec, &seat, sessions) {
4315                prompt::final_vote(&labels, &language)
4316            } else {
4317                full.clone()
4318            };
4319            vote_jobs.push(SeatJob {
4320                spec,
4321                seat,
4322                prompt: text,
4323                cwd: root.join(seat_key),
4324                timeout,
4325                allow_write: false,
4326                sessions,
4327                artifacts: artifacts.clone(),
4328                stem: format!("vote-judge-{}-recover", j + 1),
4329                handover: Some(full),
4330            });
4331            vote_pos.push(j);
4332        }
4333        let allowed = labels.clone();
4334        let mut vote_losses = Vec::new();
4335        let vote_retries = self.state.config.graph.retries;
4336        let vote_cache = self.state.config.cache_dir();
4337        let ctx = WaveCtx {
4338            carry_seats: false,
4339            pauses: Some((&self.pause, &self.interrupt)),
4340            run: &run_id,
4341            node: "vote",
4342            prompts: &prompts,
4343            cache: vote_cache.as_deref(),
4344            round: None,
4345        };
4346        let votes = ask_json_wave::<FinalVote>(
4347            vote_jobs,
4348            Arc::clone(&self.sem),
4349            vote_retries,
4350            &self.roles.judge_roster,
4351            &ctx,
4352            &mut vote_losses,
4353            &mut self.state,
4354            &move |v: &FinalVote| match v.label() {
4355                Some(c) if allowed.contains(&c) => Ok(()),
4356                other => bail!("vote {other:?} is not one of {allowed:?}"),
4357            },
4358        )
4359        .await;
4360        if self.state.resource_block.is_some() {
4361            self.stop_for_resource()?;
4362            return Ok(false);
4363        }
4364        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4365            let agent_id = seat.agent.clone();
4366            self.state.seats.insert(seat.key.clone(), seat);
4367            match res {
4368                Ok((v, _)) => {
4369                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4370                        rec.vote = v.label();
4371                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4372                    } else {
4373                        self.state.votes.push(VoteRecord {
4374                            judge: j + 1,
4375                            agent: agent_id,
4376                            vote: v.label(),
4377                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4378                            changed: false,
4379                        });
4380                    }
4381                    self.state.event(
4382                        "recover",
4383                        format!("judge {} voted again after the limit", j + 1),
4384                    );
4385                }
4386                Err(e) => {
4387                    self.state
4388                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4389                }
4390            }
4391        }
4392
4393        // A seat that ranked again is present even if its re-vote failed —
4394        // `tally` falls back to the initial ranking's first choice — so clear
4395        // its quota loss. Seats that still fail keep theirs and stay absent.
4396        let recovered_keys: BTreeSet<String> = recovered
4397            .iter()
4398            .map(|&j| format!("judge-{}", j + 1))
4399            .collect();
4400        self.state
4401            .quota
4402            .retain(|q| !recovered_keys.contains(&q.seat));
4403        // A seat that hit the limit again is a fresh loss, not the old one:
4404        // replace the stale entry so the history stays one-per-seat and the
4405        // daemon can tell this attempt's loss from a previous session's.
4406        for loss in judge_losses.into_iter().chain(vote_losses) {
4407            if recovered_keys.contains(&loss.seat) {
4408                continue;
4409            }
4410            self.state.quota.retain(|q| q.seat != loss.seat);
4411            self.state.quota.push(loss);
4412        }
4413
4414        // Recompute the verdict from the refreshed panel.
4415        self.state.tally = None;
4416        self.tally()?;
4417        Ok(self
4418            .state
4419            .tally
4420            .as_ref()
4421            .map(|t| t.met_quorum)
4422            .unwrap_or(false))
4423    }
4424
4425    // ---------------------------------------------------------------- graft
4426
4427    /// Port the strengths the judges attributed to the losing candidates onto
4428    /// the winner, once, between the tally and the review loop.
4429    ///
4430    /// Best-effort by construction: it returns nothing and fails nothing. A
4431    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4432    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4433    /// the run carries on as if the phase were off. See [`graft`] for the
4434    /// decisions and `AGENTS.md` for the invariants.
4435    ///
4436    /// The one exception is a rollback that could not be confirmed: the
4437    /// rejected commit may still be the winner's tip, so the run is blocked
4438    /// (resumable; the record stays `Running`, which makes a resume retry the
4439    /// restore) instead of reviewing, gating and merging it.
4440    async fn graft(&mut self) -> Result<()> {
4441        let Err(e) = self.graft_inner().await else {
4442            return Ok(());
4443        };
4444        if e.is::<crate::repo_lock::ResourceBlocked>() {
4445            return Err(e);
4446        }
4447        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4448            self.state.status = RunStatus::Blocked;
4449            self.state.event("graft", format!("blocked: {e:#}"));
4450            self.state.save()?;
4451            return Err(e);
4452        }
4453        // An error after the `Running` record was saved may have left the
4454        // fixer's work in the tree: the record must not stay unsettled
4455        // (a later resume would reset past the review fixes), so confirm the
4456        // restore now, or block.
4457        if let Some(rec) = self.state.graft.clone()
4458            && !rec.outcome.settled()
4459        {
4460            if let Some(winner) = self.state.winner().cloned() {
4461                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4462                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4463                        return Err(re);
4464                    }
4465                    self.state.status = RunStatus::Blocked;
4466                    self.state.event("graft", format!("blocked: {re:#}"));
4467                    self.state.save()?;
4468                    return Err(re);
4469                }
4470            }
4471            if let Some(g) = self.state.graft.as_mut() {
4472                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4473            }
4474        }
4475        self.state.event("graft", format!("skipped: {e:#}"));
4476        if let Err(e) = self.state.save() {
4477            tracing::warn!("save after graft: {e:#}");
4478        }
4479        Ok(())
4480    }
4481
4482    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4483    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4484    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4485        let _coordination = self.coordinate(wt).await?;
4486        crate::recovery::restore_repair(wt, pre)
4487            .await
4488            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4489    }
4490
4491    /// A skip is a run event, recorded once however often the node is re-entered.
4492    fn graft_skipped(&mut self, why: &str) {
4493        let message = format!("skipped: {why}");
4494        if !self
4495            .state
4496            .events
4497            .iter()
4498            .any(|e| e.node == "graft" && e.message == message)
4499        {
4500            self.state.event("graft", message);
4501        }
4502    }
4503
4504    async fn graft_inner(&mut self) -> Result<()> {
4505        // A round that was started and never finished (crash, kill): the
4506        // fixer may have left anything in the tree, so put the winner back.
4507        if let Some(rec) = self.state.graft.clone()
4508            && !rec.outcome.settled()
4509        {
4510            // Reviews exist only if the graft ended without its record being
4511            // settled (a swallowed error): the tree has moved on since, and
4512            // a reset to `pre` would discard the review fixes.
4513            if self.state.reviews.is_empty()
4514                && let Some(winner) = self.state.winner().cloned()
4515            {
4516                self.graft_restore(&winner.worktree, &rec.pre).await?;
4517            }
4518            if rec.outcome == GraftOutcome::WaitingForRestore {
4519                self.state.graft = None;
4520                self.state.save()?;
4521            } else {
4522                if let Some(g) = self.state.graft.as_mut() {
4523                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4524                }
4525                self.state.event(
4526                    "graft",
4527                    "rolled back: the previous round was interrupted; the winner is untouched",
4528                );
4529                self.state.save()?;
4530                return Ok(());
4531            }
4532        }
4533        let plan = match graft::plan(&self.state) {
4534            Ok(plan) => plan,
4535            Err(skip) => {
4536                if skip != graft::Skip::AlreadyDone {
4537                    self.graft_skipped(skip.reason());
4538                    self.state.save()?;
4539                }
4540                return Ok(());
4541            }
4542        };
4543        let Some(winner) = self.state.winner().cloned() else {
4544            return Ok(());
4545        };
4546        let wt = winner.worktree.clone();
4547        if !git::is_clean(&wt).await? {
4548            self.graft_skipped("the winner's worktree has uncommitted changes");
4549            self.state.save()?;
4550            return Ok(());
4551        }
4552        let pre = git::rev_parse(&wt, "HEAD").await?;
4553        // Saved before the fixer is called: a stop from here on is repaired
4554        // by the branch above on the next entry.
4555        self.state.graft = Some(GraftRecord {
4556            pre: pre.clone(),
4557            post: None,
4558            lines: None,
4559            limit: None,
4560            agent: None,
4561            outcome: GraftOutcome::Running,
4562            requested: graft::requested(&plan),
4563            adoption: None,
4564            adoption_error: None,
4565        });
4566        self.state.event(
4567            "graft",
4568            format!(
4569                "porting strengths of candidate(s) {} onto {}",
4570                plan.strengths
4571                    .iter()
4572                    .map(|(l, _, _)| l.to_string())
4573                    .collect::<Vec<_>>()
4574                    .join(", "),
4575                plan.winner
4576            ),
4577        );
4578        self.state.save()?;
4579
4580        let fix_prompt = prompt::graft(
4581            &self.state.instruction,
4582            plan.winner,
4583            &plan.strengths,
4584            &self.state.config.graph.language,
4585        );
4586        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4587        let sessions = self.state.config.graph.sessions;
4588        let artifacts = agent::artifacts_dir(&self.state.dir());
4589        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4590        let (_, seat, out) = self
4591            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4592                SeatJob {
4593                    prompt: fix_prompt.clone(),
4594                    spec,
4595                    seat,
4596                    cwd: wt.clone(),
4597                    timeout,
4598                    allow_write: true,
4599                    sessions,
4600                    artifacts: artifacts.clone(),
4601                    stem: "graft".to_owned(),
4602                    handover: None,
4603                }
4604            })
4605            .await;
4606        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4607            // Preserve the need to restore earlier fallback edits without
4608            // confusing an unstarted seat with an interrupted invocation.
4609            self.state.graft.as_mut().expect("graft record").outcome =
4610                GraftOutcome::WaitingForRestore;
4611            return self.stop_for_resource();
4612        }
4613        let mut why: Option<String> = None;
4614        match &out {
4615            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4616                why = Some("the fixer produced nothing usable".to_owned());
4617            }
4618            AgentOutcome::Ok(_) => {}
4619            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4620            AgentOutcome::Quota(o) => {
4621                self.state.quota.push(QuotaLoss {
4622                    seat: seat.key.clone(),
4623                    node: "graft".to_owned(),
4624                    at: Timestamp::now(),
4625                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4626                });
4627                why = Some("rate limited (quota)".to_owned());
4628            }
4629            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4630                why = Some(format!(
4631                    "the fixer failed: {}",
4632                    e.lines()
4633                        .next()
4634                        .unwrap_or("")
4635                        .chars()
4636                        .take(160)
4637                        .collect::<String>()
4638                ));
4639            }
4640        }
4641        let agent_id = seat.agent.clone();
4642        self.state.seats.insert(seat.key.clone(), seat);
4643
4644        let mut post = None;
4645        let mut lines = None;
4646        let _coordination = self.coordinate(&wt).await?;
4647        if why.is_none() {
4648            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4649            if let Ok(r) = &rescued {
4650                self.state.note_withheld("graft", &r.withheld);
4651            }
4652            let head = git::rev_parse(&wt, "HEAD").await?;
4653            // What is measured must be what gets reviewed and gated: a failed
4654            // rescue, or anything still uncommitted (withheld lockfiles
4655            // aside, which are never part of the tree), rejects the graft.
4656            let dirty = match &rescued {
4657                Ok(r) => git::status_porcelain(&wt)
4658                    .await
4659                    .map(|st| {
4660                        st.lines().any(|l| {
4661                            !r.withheld
4662                                .iter()
4663                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4664                        })
4665                    })
4666                    .unwrap_or(true),
4667                Err(_) => true,
4668            };
4669            if dirty {
4670                why = Some("the fixer left changes that could not be committed".to_owned());
4671            } else if head == pre {
4672                why = Some("the fixer changed nothing".to_owned());
4673            } else {
4674                lines = git::changed_lines(&wt, &pre, &head).await;
4675                post = Some(head);
4676            }
4677        }
4678        let (adoption, adoption_error) = match &out {
4679            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4680            _ => (None, None),
4681        };
4682        let requested = graft::requested(&plan);
4683        let outcome = match why {
4684            None => {
4685                self.state.event(
4686                    "graft",
4687                    format!(
4688                        "applied: {}",
4689                        lines.map_or_else(
4690                            || "changed lines unavailable".to_owned(),
4691                            |n| format!("{n} changed lines")
4692                        )
4693                    ),
4694                );
4695                GraftOutcome::Applied
4696            }
4697            Some(why) => {
4698                // Not settled until the restore is confirmed: the record
4699                // stays `Running`, so a resume retries it.
4700                crate::recovery::restore_repair(&wt, &pre)
4701                    .await
4702                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4703                self.state.event(
4704                    "graft",
4705                    format!("rolled back: {why}; the winner is untouched"),
4706                );
4707                GraftOutcome::RolledBack(why)
4708            }
4709        };
4710        self.state.graft = Some(GraftRecord {
4711            pre,
4712            post,
4713            lines,
4714            limit: None,
4715            agent: Some(agent_id),
4716            outcome,
4717            requested,
4718            adoption,
4719            adoption_error,
4720        });
4721        self.state.save()?;
4722        Ok(())
4723    }
4724
4725    // ----------------------------------------------------------------- fold
4726
4727    async fn fold_losers(&mut self) -> Result<()> {
4728        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4729            return Ok(());
4730        };
4731        if !self.state.candidate_preservation.is_empty() {
4732            bail!("candidate preservation incomplete; refusing cleanup");
4733        }
4734        let repo = self.state.repo.clone();
4735        let _coordination = self.coordinate(&repo).await?;
4736        let mut folded = Vec::new();
4737        for i in 0..self.state.candidates.len() {
4738            let c = &self.state.candidates[i];
4739            if c.label == winner || c.folded {
4740                continue;
4741            }
4742            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4743            remove_preserved_worktree(&repo, &wt).await?;
4744            if git::branch_exists(&repo, &branch).await?
4745                && !git::branch_delete(&repo, &branch).await?
4746            {
4747                bail!("candidate branch removal refused");
4748            }
4749            self.state.candidates[i].folded = true;
4750            self.state.save()?;
4751            folded.push(label.to_string());
4752        }
4753        // The judges are finished; their checkouts are pure cost from here.
4754        let root = self.state.worktree_root();
4755        for j in 1..=self.roles.judges.len() {
4756            let wt = root.join(format!("judge-{j}"));
4757            if wt.exists() {
4758                remove_preserved_worktree(&repo, &wt).await?;
4759            }
4760        }
4761        // The design-deliberation stage is finished by the time a tally
4762        // exists — same reasoning as the judges above.
4763        if self.state.config.graph.advise {
4764            for k in 1..=self.state.config.graph.advisors {
4765                let wt = root.join(format!("advisor-{k}"));
4766                if wt.exists() {
4767                    remove_preserved_worktree(&repo, &wt).await?;
4768                }
4769            }
4770        }
4771        if !folded.is_empty() {
4772            self.state
4773                .event("fold", format!("folded candidates {}", folded.join(", ")));
4774            self.state.save()?;
4775        }
4776        Ok(())
4777    }
4778
4779    // ------------------------------------------------------------ base sync
4780
4781    async fn hold_base_sync(&mut self) -> Result<bool> {
4782        self.hold_base_sync_as(None).await
4783    }
4784
4785    async fn hold_base_sync_kind(
4786        &mut self,
4787        kind: crate::base_sync_recovery::StopKind,
4788    ) -> Result<()> {
4789        self.hold_base_sync_as(Some(kind)).await?;
4790        Ok(())
4791    }
4792
4793    async fn hold_base_sync_as(
4794        &mut self,
4795        kind: Option<crate::base_sync_recovery::StopKind>,
4796    ) -> Result<bool> {
4797        if !self.state.base_sync_blocked() {
4798            self.state.base_sync_recovery.stop = None;
4799            return Ok(false);
4800        }
4801        let head = match self.state.winner() {
4802            Some(w) => git::rev_parse(&w.worktree, "HEAD")
4803                .await
4804                .unwrap_or_default(),
4805            None => String::new(),
4806        };
4807        self.state.record_base_sync_stop(&head);
4808        if let Some(kind) = kind {
4809            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
4810        }
4811        self.state.save()?;
4812        Ok(true)
4813    }
4814
4815    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4816    /// anything verifies it.
4817    ///
4818    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4819    /// read whatever is checked out in the winner's worktree. Left alone that
4820    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4821    /// it when the run *branched* - and a run takes long enough that the base
4822    /// has usually moved by the time it gets here. A gate that ran there
4823    /// answers "green on the commit this run started from", not "green on
4824    /// what is about to land", and the difference showed up three times in
4825    /// one day as a green run whose merge would have reverted a file another
4826    /// pull request had already landed.
4827    ///
4828    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4829    /// `land::Step::Rebase` calls, rather than a second implementation of the
4830    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4831    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4832    /// doc warns about ("two answers to a question nobody notices until a
4833    /// diff is wrong").
4834    ///
4835    /// Foreign-checkout reviews instead merge the base into the owned shadow
4836    /// worktree, preserving the ancestry required for fast-forward delivery.
4837    /// A conflicting merge is aborted and held for operator resolution.
4838    ///
4839    /// A conflict is not the end of the road: the standing rebase is handed
4840    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4841    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4842    /// land). Once it finishes, review and the gate run as usual on the
4843    /// rebased tree, which is where a breakage the new base caused is caught
4844    /// by the ordinary gate-fix round. magi resolves nothing itself.
4845    ///
4846    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4847    /// `state.base_sync.attempts`, is how many times the base is *rebased
4848    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4849    /// *conflict* was given to a fixer. When the fixer cannot finish the
4850    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4851    /// what was tried (rounds spent, paths still conflicted) and the branch
4852    /// and worktree stay exactly as they were - untouched, for a person to
4853    /// look at - which is also what makes re-entering this function
4854    /// afterwards a no-op instead of a second attempt at the same wall. A
4855    /// push failure ends the same way.
4856    async fn sync_to_base(&mut self) -> Result<()> {
4857        if self.state.status == RunStatus::AlreadyInBase {
4858            return Ok(());
4859        }
4860        self.hold_base_sync().await?;
4861        let conflicted = self
4862            .state
4863            .base_sync
4864            .as_ref()
4865            .is_some_and(|s| s.conflict.is_some());
4866        let Some(winner) = self.state.winner().cloned() else {
4867            return Ok(());
4868        };
4869
4870        let repo = self.state.repo.clone();
4871        let remote = self.state.config.merge.remote.clone();
4872        let base_branch = self.state.base_branch.clone();
4873        let tracking = format!("{remote}/{base_branch}");
4874
4875        let fetched_base =
4876            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
4877        // No network, or the remote never had this branch: the run already
4878        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4879        // otherwise), and one that got this far is not blocked by a fetch
4880        // that fails now. It just has no newer tip to compare against.
4881        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4882            return Ok(());
4883        };
4884
4885        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4886        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4887        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4888
4889        // Before any rebase, and before a recorded conflict is honoured: a
4890        // branch whose change reached the base under other commit ids has
4891        // nothing to rebase and nothing to conflict with, and a run that
4892        // already stopped on that phantom conflict recovers here on resume.
4893        // `behind == 0` with head == tip is a branch the base has since taken
4894        // in whole, whether or not a conflict was ever recorded: the ancestry
4895        // proof must still run (`classify` ignores a head still on the start
4896        // commit).
4897        if (behind > 0 || conflicted || head == tip)
4898            && self
4899                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4900                .await?
4901        {
4902            return Ok(());
4903        }
4904        if conflicted && self.state.base_recovery_pending() {
4905            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
4906            let owner =
4907                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
4908            if !fetched_base
4909                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
4910                || !git::is_clean(&winner.worktree).await?
4911                || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
4912            {
4913                return Ok(());
4914            }
4915        }
4916        let recovery = conflicted && self.state.consume_base_recovery(&head);
4917        if conflicted {
4918            // Consumption (including a refused changed HEAD) is durable before Git.
4919            self.state.save()?;
4920            if !recovery {
4921                return Ok(());
4922            }
4923        }
4924
4925        if behind == 0 {
4926            // A fixer-finished rebase moves the branch ref before the
4927            // winner's worktree is told (`sync_to_head` below). A run that
4928            // died in between resumes here with `behind == 0` and a tree still
4929            // holding the pre-rebase files, which review and the gate would
4930            // then read. That state is exactly: HEAD moved off the tip the
4931            // rebase started from, yet the tree is still identical to that
4932            // tip. A tree with edits of its own differs from it, so nothing
4933            // is thrown away.
4934            if let Some(from) = self
4935                .state
4936                .rebase_fixes
4937                .iter()
4938                .rev()
4939                .find_map(|r| r.from.clone())
4940                && from != head
4941                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4942                    .await
4943                    .is_ok_and(|o| o.ok())
4944            {
4945                git::sync_to_head(&winner.worktree).await?;
4946            }
4947            // An earlier attempt may have rebased the branch locally and died
4948            // before pushing it (only the fresh-rebase arm below pushes).
4949            // Publish it now, so the plain push at PR time is not refused as
4950            // a non-fast-forward. A run already holding a recorded conflict
4951            // never reaches here; that case is out of scope.
4952            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4953            if let Some(why) = &conflict {
4954                self.state.status = RunStatus::Blocked;
4955                self.state.event("land", why.clone());
4956            }
4957            self.state.base_sync = Some(BaseSync {
4958                tip,
4959                behind: 0,
4960                attempts: attempts + usize::from(recovery),
4961                conflict,
4962                already_in: None,
4963            });
4964            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4965                .await?;
4966            self.state.save()?;
4967            return Ok(());
4968        }
4969
4970        if attempts >= BASE_SYNC_ROUNDS && !recovery {
4971            let why = format!(
4972                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4973                 synchronization(s); synchronizing again would only race it",
4974                winner.branch
4975            );
4976            self.state.status = RunStatus::Blocked;
4977            self.state.base_sync = Some(BaseSync {
4978                tip,
4979                behind,
4980                attempts,
4981                conflict: Some(why.clone()),
4982                already_in: None,
4983            });
4984            self.state.event("land", why);
4985            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
4986                .await?;
4987            self.state.save()?;
4988            return Ok(());
4989        }
4990
4991        if self.state.foreign_checkout.is_some() {
4992            // Delivery to the operator's checkout is fast-forward only. A
4993            // rebase here would discard its saved head from our ancestry.
4994            let wait = repository_wait(&self.state, None);
4995            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
4996            let merged = git::git_raw(
4997                &winner.worktree,
4998                &[
4999                    "-c",
5000                    "core.hooksPath=/dev/null",
5001                    "merge",
5002                    "--no-ff",
5003                    "--no-autostash",
5004                    "--no-edit",
5005                    "-m",
5006                    &format!("Merge {tracking} before verifying"),
5007                    &tip,
5008                ],
5009            )
5010            .await?;
5011            let conflict = if merged.ok() {
5012                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5013                self.state.event(
5014                    "land",
5015                    format!("merged {tracking} into {} before verifying", winner.branch),
5016                );
5017                None
5018            } else {
5019                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5020                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5021                }
5022                let why = format!(
5023                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5024                    winner.branch,
5025                    merged.stderr.chars().take(600).collect::<String>()
5026                );
5027                self.state.status = RunStatus::Blocked;
5028                self.state.event("land", &why);
5029                Some(why)
5030            };
5031            let synced = conflict.is_none();
5032            self.state.base_sync = Some(BaseSync {
5033                tip,
5034                behind: if synced { 0 } else { behind },
5035                attempts: attempts + 1,
5036                conflict,
5037                already_in: None,
5038            });
5039            self.state.save()?;
5040            return Ok(());
5041        }
5042
5043        self.state.event(
5044            "land",
5045            format!(
5046                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5047                winner.branch
5048            ),
5049        );
5050        self.state.save()?;
5051
5052        // The remote's copy of the branch, read now and only if the fetch
5053        // really succeeded (a stale tracking ref must never pin a lease). It is
5054        // pushed over after a rebase only when it is a commit this branch
5055        // already contains, by ancestry or by patch (an earlier rebase of ours
5056        // that never reached the remote): anything else is somebody else's work.
5057        let branch_tracking = format!("{remote}/{}", winner.branch);
5058        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5059        let remote_tip = if self.state.task_pr_branch().is_none()
5060            && matches!(&fetched_branch, Ok(o) if o.ok())
5061        {
5062            git::rev_parse(&repo, &branch_tracking).await.ok()
5063        } else {
5064            None
5065        };
5066        // A remote tip this branch does not contain is somebody else's work:
5067        // rebasing would leave a local tip that can never be pushed. Stop
5068        // before touching anything and say so.
5069        if let Some(theirs) = &remote_tip
5070            && !git::is_ancestor(&repo, theirs, &head).await
5071            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5072                .await
5073                .is_ok_and(|missing| missing.is_empty())
5074        {
5075            let why = format!(
5076                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5077                 them",
5078                short(theirs),
5079                winner.branch
5080            );
5081            self.state.status = RunStatus::Blocked;
5082            self.state.base_sync = Some(BaseSync {
5083                tip,
5084                behind,
5085                attempts: attempts + usize::from(recovery),
5086                conflict: Some(why.clone()),
5087                already_in: None,
5088            });
5089            self.state.event("land", why);
5090            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5091                .await?;
5092            self.state.save()?;
5093            return Ok(());
5094        }
5095
5096        let scratch = self.state.dir().join("base-sync");
5097        let rebased =
5098            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5099                .await
5100            {
5101                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5102                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5103                Err(e) => Err(e),
5104            };
5105        let attempts = attempts + 1;
5106        let stop_kind = match &rebased {
5107            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5108            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5109            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5110        };
5111        match rebased {
5112            Ok(None) => {
5113                // The branch ref moved, but a worktree that already had it
5114                // checked out (the winner's) was not told; sync its index and
5115                // files before anything reads them.
5116                git::sync_to_head(&winner.worktree).await?;
5117                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5118                let mut conflict = None;
5119                if let Some(pinned) = &remote_tip {
5120                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5121                    match pushed {
5122                        Ok(o) if o.ok() => self.state.event(
5123                            "land",
5124                            format!("pushed rebased {} to {remote}", winner.branch),
5125                        ),
5126                        Ok(o) => {
5127                            conflict = Some(format!(
5128                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5129                                winner.branch,
5130                                short(pinned),
5131                                o.stderr.chars().take(600).collect::<String>()
5132                            ));
5133                        }
5134                        Err(e) => {
5135                            conflict = Some(format!(
5136                                "rebased {} locally but could not push it: {e:#}",
5137                                winner.branch
5138                            ));
5139                        }
5140                    }
5141                }
5142                if let Some(why) = &conflict {
5143                    self.state.status = RunStatus::Blocked;
5144                    self.state.event("land", why.clone());
5145                }
5146                self.state.base_sync = Some(BaseSync {
5147                    tip: tip.clone(),
5148                    behind: 0,
5149                    attempts,
5150                    conflict,
5151                    already_in: None,
5152                });
5153                self.state
5154                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
5155            }
5156            Ok(Some(conflict)) => {
5157                let why = format!(
5158                    "{} conflicts with {tracking} and did not rebase: {}",
5159                    winner.branch,
5160                    conflict.chars().take(600).collect::<String>()
5161                );
5162                self.state.status = RunStatus::Blocked;
5163                self.state.base_sync = Some(BaseSync {
5164                    tip,
5165                    behind,
5166                    attempts,
5167                    conflict: Some(why.clone()),
5168                    already_in: None,
5169                });
5170                self.state.event("land", why);
5171            }
5172            // A refused Git write or a busy repository: nothing is wrong with
5173            // the branch. The scratch rebase stays standing and the run
5174            // resumes into it (`base_sync` must not claim a conflict).
5175            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5176                self.state.save()?;
5177                return Ok(());
5178            }
5179            Err(e) => {
5180                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5181                self.state.status = RunStatus::Blocked;
5182                self.state.base_sync = Some(BaseSync {
5183                    tip,
5184                    behind,
5185                    attempts,
5186                    conflict: Some(why.clone()),
5187                    already_in: None,
5188                });
5189                self.state.event("land", why);
5190            }
5191        }
5192        if !self.state.base_sync_blocked() {
5193            self.state.base_sync_recovery.stop = None;
5194        }
5195        self.hold_base_sync_kind(stop_kind).await?;
5196        self.state.save()?;
5197        Ok(())
5198    }
5199
5200    /// Push a branch an earlier attempt rebased locally but never published,
5201    /// pinned to the remote tip read right after a successful fetch. Returns
5202    /// the reason when the run must stop; `None` when there was nothing to do
5203    /// (no remote copy, the same tip, or a remote copy this branch already
5204    /// contains, which the PR-time push fast-forwards) or the push succeeded.
5205    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5206        // Task competitions publish only after the PR text gate, to their
5207        // public head; never resurrect a candidate branch on the remote.
5208        if self.state.task_pr_branch().is_some() {
5209            return None;
5210        }
5211        let repo = self.state.repo.clone();
5212        let remote = self.state.config.merge.remote.clone();
5213        let fetched = git::fetch(&repo, &remote, branch).await;
5214        if !matches!(&fetched, Ok(o) if o.ok()) {
5215            return None;
5216        }
5217        let branch_tracking = format!("{remote}/{branch}");
5218        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5219        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5220            return None;
5221        }
5222        if !crate::reconcile::origin_missing(&repo, head, &theirs)
5223            .await
5224            .is_ok_and(|missing| missing.is_empty())
5225        {
5226            return Some(format!(
5227                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5228                 them",
5229                short(&theirs)
5230            ));
5231        }
5232        match git::push_pinned(&repo, &remote, branch, &theirs).await {
5233            Ok(o) if o.ok() => {
5234                self.state
5235                    .event("land", format!("pushed rebased {branch} to {remote}"));
5236                None
5237            }
5238            Ok(o) => Some(format!(
5239                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5240                 someone may have pushed): {}",
5241                short(&theirs),
5242                o.stderr.chars().take(600).collect::<String>()
5243            )),
5244            Err(e) => Some(format!(
5245                "{branch} is rebased locally but could not be pushed: {e:#}"
5246            )),
5247        }
5248    }
5249
5250    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
5251    /// is already on `tip` under other commit ids ([`crate::already`]); returns
5252    /// whether it did.
5253    ///
5254    /// Checked only when the base is ahead of the branch. A failing check is
5255    /// "not proven" - the ordinary rebase path then decides - never a reason to
5256    /// stop the run.
5257    ///
5258    /// The remote copy of the branch is held to the same standard as the local
5259    /// one: if it carries a tip this worktree does not, that tip must itself be
5260    /// proven in the base, or nothing is settled (a pull request would
5261    /// otherwise be closed over commits nobody checked). The pull request is
5262    /// closed *before* the terminal status is saved; if that fails for a
5263    /// reason other than a refusal (no network, a `gh` error) the run is left
5264    /// `Blocked` with the reason as its conflict, which a resume retries -
5265    /// the same recovery a phantom conflict gets.
5266    async fn settle_already_in(
5267        &mut self,
5268        branch: &str,
5269        tip: &str,
5270        head: &str,
5271        attempts: usize,
5272        behind: usize,
5273    ) -> Result<bool> {
5274        let repo = self.state.repo.clone();
5275        let remote = self.state.config.merge.remote.clone();
5276        let start = self.state.base_commit.clone();
5277        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5278            Ok(Some(e)) => e,
5279            Ok(None) => return Ok(false),
5280            Err(e) => {
5281                tracing::warn!("already-in-base check for {branch}: {e:#}");
5282                return Ok(false);
5283            }
5284        };
5285        let mut verified = vec![head.to_owned()];
5286        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5287        let fetched = git::fetch(&repo, &remote, &public_branch).await;
5288        if matches!(&fetched, Ok(o) if o.ok())
5289            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5290            && theirs != head
5291        {
5292            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5293                Ok(Some(_)) => verified.push(theirs),
5294                _ => return Ok(false),
5295            }
5296        }
5297        let base_branch = self.state.base_branch.clone();
5298        let message = format!(
5299            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5300             land",
5301            evidence.names(),
5302            evidence.proof.as_str()
5303        );
5304        let closed =
5305            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5306                .await;
5307        match closed {
5308            Ok(Ok(url)) => self
5309                .state
5310                .event("land", format!("closed {url}: superseded on {base_branch}")),
5311            Ok(Err(why)) => self
5312                .state
5313                .event("land", format!("did not close a pull request: {why}")),
5314            Err(e) => {
5315                let why = format!(
5316                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
5317                     not be closed ({e:#}); resume to retry"
5318                );
5319                self.state.status = RunStatus::Blocked;
5320                self.state.base_sync = Some(BaseSync {
5321                    tip: tip.to_owned(),
5322                    behind,
5323                    attempts,
5324                    conflict: Some(why.clone()),
5325                    already_in: None,
5326                });
5327                self.state.event("land", why);
5328                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5329                    .await?;
5330                self.state.save()?;
5331                return Ok(true);
5332            }
5333        }
5334        self.state.status = RunStatus::AlreadyInBase;
5335        self.state.base_sync_recovery.stop = None;
5336        self.state.base_sync = Some(BaseSync {
5337            tip: tip.to_owned(),
5338            behind,
5339            attempts,
5340            conflict: None,
5341            already_in: Some(evidence),
5342        });
5343        self.state.event("land", message);
5344        self.state.save()?;
5345        self.settle_questions();
5346        Ok(true)
5347    }
5348
5349    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
5350    /// last landed the winner on, once it has run, else the commit the run
5351    /// branched from.
5352    ///
5353    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
5354    /// and `vote` all happen before there is a winner to rebase, so they
5355    /// compare every candidate against the branch point on purpose, and a
5356    /// base that moves after they are already done cannot change an answer
5357    /// they already gave.
5358    fn landing_base(&self) -> String {
5359        self.state
5360            .base_sync
5361            .as_ref()
5362            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5363    }
5364
5365    // ------------------------------------------------------- operator fix
5366
5367    /// Route specific, already-recorded review findings to a fixer for a
5368    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5369    /// entry point.
5370    ///
5371    /// Distinct from `review_loop`'s own fix step in three ways: it never
5372    /// runs a reviewer wave, it never spends review-round budget, and what
5373    /// happened is recorded as an [`OperatorFixRequest`] appended to
5374    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5375    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5376    /// and vote must never be rewritten to look like a manufactured blocking
5377    /// verdict.
5378    ///
5379    /// Only meaningful once review has actually concluded: `Ready` (handed
5380    /// off with findings still open, or simply concluded clean while minor
5381    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5382    /// gate failed). Everything else is refused: a run still in progress
5383    /// should simply be resumed, and a `Merged` run's branch has already
5384    /// landed — reopening *this* run's own record cannot change that, so the
5385    /// answer there is a fresh `magi review <branch>`.
5386    ///
5387    /// A real commit here re-verifies through a fresh, ordinary review-only
5388    /// run on the same branch ([`Self::review`]) rather than reopening this
5389    /// run's own `review_loop`: once any round in this run's history went
5390    /// clean, `review_conclusion` treats that as permanent by design (the
5391    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5392    /// way to force one more genuine reviewer wave out of *this* run without
5393    /// either rewriting history or weakening that guarantee for every other
5394    /// caller. A review-only run costs nothing extra — no implementation, no
5395    /// judging, no vote — and exercises the exact same review → verify →
5396    /// gate → (human) merge path, unmodified.
5397    pub async fn fix_selected(
5398        &mut self,
5399        ids: &[String],
5400        reason: &str,
5401        allow_stale: bool,
5402    ) -> Result<()> {
5403        let reason = reason.trim();
5404        if reason.is_empty() {
5405            bail!("a fix request needs a reason — that is the operator's own record of why");
5406        }
5407        if ids.is_empty() {
5408            bail!("no finding id given");
5409        }
5410        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5411            bail!(
5412                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5413                 has already concluded — can be given a targeted fix. A run still \
5414                 in progress should simply be resumed; a `merged` run's branch has \
5415                 already landed, so its answer is a fresh `magi review <branch>`, \
5416                 not reopening this run's own record",
5417                self.state.id,
5418                self.state.status.as_str()
5419            );
5420        }
5421        let Some(winner) = self.state.winner().cloned() else {
5422            bail!("run {} has no winning candidate to fix", self.state.id);
5423        };
5424        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5425            bail!(
5426                "branch `{}` no longer exists; this run cannot be extended",
5427                winner.branch
5428            );
5429        }
5430        let home = crate::run::home();
5431        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5432            bail!(
5433                "run {} is currently being worked on by another magi process",
5434                self.state.id
5435            );
5436        }
5437        // Held for the rest of this call, including the follow-up review
5438        // below: two `magi fix` invocations against the same run must not
5439        // both reach the worktree manipulation further down, which would
5440        // otherwise race to remove and recreate the same directory — see
5441        // [`FixClaim`]'s own doc.
5442        let _claim = FixClaim::acquire(&self.state.dir())?;
5443
5444        // Resolve every id before spending anything — an unknown id refuses
5445        // the whole request rather than silently dropping it — and dedup
5446        // while keeping the operator's own order.
5447        let mut seen = BTreeSet::new();
5448        let mut findings = Vec::new();
5449        let mut missing = Vec::new();
5450        for id in ids {
5451            if !seen.insert(id.clone()) {
5452                continue;
5453            }
5454            match self.state.finding(id) {
5455                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5456                    id: f.id.clone(),
5457                    severity: f.severity,
5458                    reviewer_vote: rec.vote,
5459                    round: round.round,
5460                    round_head: round.head.clone(),
5461                    reviewer: rec.reviewer,
5462                    agent: rec.agent.clone(),
5463                    file: f.file.clone(),
5464                    line: f.line,
5465                    title: f.title.clone(),
5466                    detail: f.detail.clone(),
5467                    outcome: OperatorFixOutcome::Pending,
5468                }),
5469                None => missing.push(id.clone()),
5470            }
5471        }
5472        if !missing.is_empty() {
5473            bail!(
5474                "unknown finding id(s): {}; nothing was changed",
5475                missing.join(", ")
5476            );
5477        }
5478
5479        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5480        let stale_details: Vec<(String, String)> = findings
5481            .iter()
5482            .filter(|f| f.round_head != head_at_request)
5483            .map(|f| (f.id.clone(), f.round_head.clone()))
5484            .collect();
5485        let stale = !stale_details.is_empty();
5486        if stale && !allow_stale {
5487            bail!(
5488                "the branch has moved since some finding(s) were raised — {} — now \
5489                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5490                stale_details
5491                    .iter()
5492                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5493                    .collect::<Vec<_>>()
5494                    .join(", "),
5495                short(&head_at_request)
5496            );
5497        }
5498
5499        let request = OperatorFixRequest {
5500            requested_at: Timestamp::now(),
5501            reason: reason.to_owned(),
5502            findings,
5503            head_at_request: head_at_request.clone(),
5504            allow_stale,
5505            stale,
5506            fix: None,
5507            result_head: None,
5508            follow_up_review_run: None,
5509        };
5510        self.state.event(
5511            "fix",
5512            format!(
5513                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5514                request.findings.len(),
5515                request
5516                    .findings
5517                    .iter()
5518                    .map(|f| f.id.as_str())
5519                    .collect::<Vec<_>>()
5520                    .join(", "),
5521            ),
5522        );
5523        // Recorded now, before any worktree work or the fixer call itself —
5524        // and re-saved at each checkpoint below: a crash at any point after
5525        // this (mid fixer call, mid follow-up review) must not lose the fact
5526        // that this was requested, for which findings, and why. Everything
5527        // past this point reads and writes through `request_index` rather
5528        // than a local variable, since `request` itself is moved here.
5529        self.state.operator_fixes.push(request);
5530        self.state.save()?;
5531        let request_index = self.state.operator_fixes.len() - 1;
5532
5533        // A fresh, dedicated worktree for this one call, never the winner's
5534        // own worktree in place: that one may already be gone (folded away),
5535        // and reusing it in place would leave the branch checked out there
5536        // when the follow-up review below tries to check it out again. Freed
5537        // immediately after, either way — but only once confirmed clean:
5538        // `worktree_remove` is a `git worktree remove --force`, which would
5539        // otherwise discard uncommitted work left there by the operator or
5540        // another process before this had a chance to even look at it.
5541        let repo = self.state.repo.clone();
5542        let coordination = self.coordinate(&repo).await?;
5543        if winner.worktree.exists() {
5544            // Lockfiles a rescue commit withheld stay untracked on purpose and
5545            // are already recorded; they are not the operator's work to protect.
5546            let dirty = git::git(
5547                &winner.worktree,
5548                &["status", "--porcelain", "--untracked-files=all"],
5549            )
5550            .await?;
5551            let only_withheld = dirty.lines().all(|l| {
5552                l.strip_prefix("?? ")
5553                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5554            });
5555            if !only_withheld {
5556                bail!(
5557                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5558                     discard them first",
5559                    winner.worktree.display()
5560                );
5561            }
5562            git::worktree_remove(&self.state.repo, &winner.worktree)
5563                .await
5564                .ok();
5565        }
5566        let fix_worktree = self.state.worktree_root().join("operator-fix");
5567        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5568        git::git(
5569            &self.state.repo,
5570            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5571        )
5572        .await
5573        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5574        if !git::is_clean(&fix_worktree).await? {
5575            git::worktree_remove(&self.state.repo, &fix_worktree)
5576                .await
5577                .ok();
5578            bail!(
5579                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5580                winner.branch
5581            );
5582        }
5583        let repo_for_setup = self.state.repo.clone();
5584        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5585            Ok(()) => {
5586                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5587            }
5588            Err(e) => Err(e),
5589        };
5590        if let Err(e) = setup {
5591            git::worktree_remove(&self.state.repo, &fix_worktree)
5592                .await
5593                .ok();
5594            return Err(e);
5595        }
5596
5597        let run_id = self.state.id.clone();
5598        let prompts = self.state.config.prompts.clone();
5599        let language = self.state.config.graph.language.clone();
5600        let sessions = self.state.config.graph.sessions;
5601        let artifacts = agent::artifacts_dir(&self.state.dir());
5602        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5603            .findings
5604            .iter()
5605            .map(|f| Finding {
5606                id: f.id.clone(),
5607                severity: f.severity,
5608                file: f.file.clone(),
5609                line: f.line,
5610                title: f.title.clone(),
5611                detail: f.detail.clone(),
5612            })
5613            .collect();
5614        let fix_prompt = prompt::operator_fix_in(
5615            &self.state.instruction,
5616            &finding_list,
5617            reason,
5618            &stale_details,
5619            &head_at_request,
5620            &language,
5621            &self.state.config.graph.github_language,
5622        );
5623        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5624        drop(coordination);
5625        let (job, seat, out) = self
5626            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5627                prompt: fix_prompt.clone(),
5628                spec,
5629                seat,
5630                cwd: fix_worktree.clone(),
5631                timeout,
5632                allow_write: true,
5633                sessions,
5634                artifacts: artifacts.clone(),
5635                stem: "operator-fix".to_owned(),
5636                handover: None,
5637            })
5638            .await;
5639        let agent_id = seat.agent.clone();
5640
5641        let mut fix = FixRecord {
5642            agent: agent_id,
5643            addressed: Vec::new(),
5644            rejected: Vec::new(),
5645            notes: String::new(),
5646            committed: false,
5647            failed: None,
5648            duration_ms: 0,
5649            continuation: None,
5650        };
5651        let mut final_seat = seat.clone();
5652        match out {
5653            AgentOutcome::Ok(o) => {
5654                fix.duration_ms = o.duration_ms;
5655                let parsed = verdict::extract_json::<FixReport>(&o.text);
5656                let incomplete_reason = match &parsed {
5657                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5658                        "the reply parsed, but it reported a command whose own CLI \
5659                         never confirmed an exit status"
5660                            .to_owned(),
5661                    ),
5662                    Ok(_) => None,
5663                    Err(e) => Some(e.to_string()),
5664                };
5665                match incomplete_reason {
5666                    None => {
5667                        let report = parsed.expect("checked Ok above");
5668                        fix.addressed = report.addressed;
5669                        fix.rejected = report.rejected;
5670                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5671                    }
5672                    Some(reason) => {
5673                        let (resumed_seat, resolved, failure, cont) = self
5674                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5675                            .await;
5676                        fix.duration_ms += cont.cumulative_wait_ms;
5677                        fix.continuation = Some(cont);
5678                        final_seat = resumed_seat;
5679                        match resolved {
5680                            Some(report) => {
5681                                fix.addressed = report.addressed;
5682                                fix.rejected = report.rejected;
5683                                fix.notes =
5684                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5685                            }
5686                            None => fix.failed = failure,
5687                        }
5688                    }
5689                }
5690            }
5691            AgentOutcome::Dropped(o) => {
5692                fix.duration_ms = o.duration_ms;
5693                let why = o
5694                    .dropped
5695                    .as_ref()
5696                    .map(|d| d.why.as_str())
5697                    .unwrap_or("the CLI ended the stream without delivering its answer");
5698                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5699            }
5700            AgentOutcome::Quota(o) => {
5701                self.state.quota.push(QuotaLoss {
5702                    seat: final_seat.key.clone(),
5703                    node: "fix".to_owned(),
5704                    at: Timestamp::now(),
5705                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5706                });
5707                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5708            }
5709            AgentOutcome::Failed(e) => fix.failed = Some(e),
5710            AgentOutcome::ResourceBlocked(e) => {
5711                fix.failed = Some(format!("the fixer was not started: {e}"));
5712            }
5713        }
5714        if fix.continuation.is_none() {
5715            fix.continuation = Some(ContinuationRecord::not_needed());
5716        }
5717        self.state.seats.insert(final_seat.key.clone(), final_seat);
5718
5719        let rescue_message = format!(
5720            "magi: operator-selected fix ({}) (uncommitted work)",
5721            self.state.operator_fixes[request_index]
5722                .findings
5723                .iter()
5724                .map(|f| f.id.as_str())
5725                .collect::<Vec<_>>()
5726                .join(", ")
5727        );
5728        let repo = self.state.repo.clone();
5729        let coordination = self.coordinate(&repo).await?;
5730        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5731            self.state.note_withheld("fix", &r.withheld);
5732        }
5733        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5734        fix.committed = after != head_at_request;
5735        git::worktree_remove(&self.state.repo, &fix_worktree)
5736            .await
5737            .ok();
5738        drop(coordination);
5739
5740        self.state.event(
5741            "fix",
5742            match &fix.failed {
5743                Some(reason) => format!(
5744                    "operator fix: adoption report was lost ({reason}); {}",
5745                    if fix.committed {
5746                        "committed"
5747                    } else {
5748                        "NO new commit"
5749                    }
5750                ),
5751                None => format!(
5752                    "operator fix: {} addressed, {} rejected, {}",
5753                    fix.addressed.len(),
5754                    fix.rejected.len(),
5755                    if fix.committed {
5756                        "committed"
5757                    } else {
5758                        "NO new commit"
5759                    }
5760                ),
5761            },
5762        );
5763
5764        // Every selected finding gets an outcome — never left `Pending` once
5765        // the fixer's own turn is over. A report that never came back at all
5766        // marks every one of them `Unreported`, not silently "not addressed":
5767        // quota, a dropped stream, or an exhausted continuation are gaps in
5768        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5769        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5770        for f in &mut self.state.operator_fixes[request_index].findings {
5771            f.outcome = if fix.failed.is_some() {
5772                OperatorFixOutcome::Unreported
5773            } else if fix.addressed.contains(&f.id) {
5774                OperatorFixOutcome::Addressed
5775            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5776                OperatorFixOutcome::Rejected { why: r.why.clone() }
5777            } else {
5778                OperatorFixOutcome::Unreported
5779            };
5780        }
5781
5782        let committed = fix.committed;
5783        if committed {
5784            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5785        }
5786        self.state.operator_fixes[request_index].fix = Some(fix);
5787        // Saved again now that the fixer's own outcome is final, on top of
5788        // the save right after the request was first pushed above.
5789        self.state.save()?;
5790
5791        if committed {
5792            self.state.event(
5793                "fix",
5794                format!(
5795                    "operator fix committed {}; opening a follow-up review-only run",
5796                    short(&after)
5797                ),
5798            );
5799            // The operator asked for the fix, and the follow-up serves whatever
5800            // task the run it follows served.
5801            let origin =
5802                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5803            match Self::review(
5804                &self.state.repo,
5805                &winner.branch,
5806                self.state.config.clone(),
5807                origin,
5808            )
5809            .await
5810            {
5811                Ok(mut follow_up) => {
5812                    follow_up.state.event(
5813                        "start",
5814                        format!(
5815                            "requested by an operator fix on run {} for finding(s) {}",
5816                            self.state.id,
5817                            self.state.operator_fixes[request_index]
5818                                .findings
5819                                .iter()
5820                                .map(|f| f.id.as_str())
5821                                .collect::<Vec<_>>()
5822                                .join(", "),
5823                        ),
5824                    );
5825                    follow_up.state.save()?;
5826                    let follow_up_id = follow_up.state.id.clone();
5827                    // The follow-up is a run like any other: it belongs to the
5828                    // task of the run it follows, or to one filed for it.
5829                    let adopted = match crate::direct::adopt(
5830                        &follow_up.state,
5831                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5832                    ) {
5833                        Ok(a) => a,
5834                        Err(e) => {
5835                            // An ownerless run must not spend agent calls; it
5836                            // stays saved, and `magi run --resume` adopts it.
5837                            self.state.event(
5838                                "fix",
5839                                format!(
5840                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5841                                ),
5842                            );
5843                            self.state.save()?;
5844                            return Ok(());
5845                        }
5846                    };
5847                    let executed = follow_up.execute().await;
5848                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5849                    if let Some(a) = adopted {
5850                        a.finish(&follow_up.state, executed);
5851                    }
5852                    if let Some(e) = failure {
5853                        self.state.event(
5854                            "fix",
5855                            format!(
5856                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5857                            ),
5858                        );
5859                    }
5860                    self.state.operator_fixes[request_index].follow_up_review_run =
5861                        Some(follow_up_id);
5862                }
5863                Err(e) => {
5864                    self.state.event(
5865                        "fix",
5866                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5867                    );
5868                }
5869            }
5870            self.state.save()?;
5871        }
5872
5873        Ok(())
5874    }
5875
5876    // --------------------------------------------------------------- review
5877
5878    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5879    /// names, in order, each at most once, moving on only when the call
5880    /// advances (an error, a quota hit or nothing usable - the same decision
5881    /// point `agent::chain_advances` is for the other chained roles).
5882    ///
5883    /// Each agent gets its own seat from `Runner::seat`: the same agent
5884    /// continues its conversation (the winner's own implementer seat when it
5885    /// is the winner's author, now that the competition is over), another
5886    /// takes a fresh one so the full prompt is sent again. A handover is
5887    /// recorded under `node`, which is what makes the fallback stick for the
5888    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5889    /// is returned, so an exhausted chain reads exactly like a single failed
5890    /// fixer: one quota loss, the same wording, the same refund. An earlier
5891    /// attempt's edits are left in the tree and judged, with the final
5892    /// attempt's, by what git says afterwards.
5893    ///
5894    /// Returns the job that produced the outcome, for `continue_fix_report`.
5895    async fn ask_fixer(
5896        &mut self,
5897        winner: &Candidate,
5898        node: &'static str,
5899        round: Option<usize>,
5900        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5901    ) -> (SeatJob, SeatState, AgentOutcome) {
5902        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5903        self.ask_fixer_among(attempts, node, round, None, build)
5904            .await
5905    }
5906
5907    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5908    /// worktree and a commit: before the next agent is asked after one that
5909    /// did not satisfy the call, the worktree is put back on that commit, so
5910    /// a failed agent's partial edits are not carried into the next one.
5911    async fn ask_fixer_among(
5912        &mut self,
5913        attempts: Vec<(AgentSpec, String)>,
5914        node: &'static str,
5915        round: Option<usize>,
5916        restore: Option<(&Path, &str)>,
5917        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5918    ) -> (SeatJob, SeatState, AgentOutcome) {
5919        let run_id = self.state.id.clone();
5920        let prompts = self.state.config.prompts.clone();
5921        let cache = self.state.config.cache_dir();
5922        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5923        let mut last = None;
5924        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5925            let seat = self.seat(&key, &spec.id);
5926            let mut job = build(spec.clone(), seat);
5927            if i > 0 {
5928                job.stem = format!("{}-{}", job.stem, spec.id);
5929            }
5930            let ctx = WaveCtx {
5931                carry_seats: false,
5932                pauses: Some((&self.pause, &self.interrupt)),
5933                run: &run_id,
5934                node,
5935                prompts: &prompts,
5936                cache: cache.as_deref(),
5937                round,
5938            };
5939            let (seat, out) =
5940                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5941            // A resource block never advances the chain: the chosen agent
5942            // was not asked, so it must be the one asked again on resume.
5943            let advances = match &out {
5944                AgentOutcome::Ok(o) => agent::output_advances(o),
5945                AgentOutcome::ResourceBlocked(_) => false,
5946                _ => true,
5947            };
5948            if let Some(next) = ids.get(i + 1)
5949                && advances
5950            {
5951                self.state.seats.insert(seat.key.clone(), seat.clone());
5952                let class =
5953                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5954                let reason = match &out {
5955                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5956                    other => fail_reason(other),
5957                };
5958                // A failed agent's edits must not reach the next one, and a
5959                // restore that did not work ends the chain: the call fails
5960                // and the caller's own (confirmed) restore decides.
5961                if let Some((wt, rev)) = restore
5962                    && let Err(e) = self.graft_restore(wt, rev).await
5963                {
5964                    let why =
5965                        format!("could not restore the worktree before the next fixer: {e:#}");
5966                    tracing::warn!("{why}");
5967                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5968                        AgentOutcome::ResourceBlocked(why)
5969                    } else {
5970                        AgentOutcome::Failed(why)
5971                    };
5972                    last = Some((job, seat, out));
5973                    break;
5974                }
5975                record_handover(
5976                    &mut self.state,
5977                    node,
5978                    &seat.key,
5979                    &spec.id,
5980                    next,
5981                    &class,
5982                    &reason,
5983                );
5984                continue;
5985            }
5986            last = Some((job, seat, out));
5987            break;
5988        }
5989        last.expect("the fixer chain always has an entry")
5990    }
5991
5992    /// The one repair a verification-only failure earns (see
5993    /// [`crate::recovery`]), before the first review round of a run that took
5994    /// a branch over.
5995    ///
5996    /// Saved as started *before* the fixer is called, so a resume never
5997    /// repairs twice; a start found on re-entry is rolled back to the commit
5998    /// it began from. A failed or unusable fixer leaves the tree as it was and
5999    /// the review goes on over it, so the first round's own verification
6000    /// reports the failure again. The failure is never carried into a round as
6001    /// a result: verification is re-run on whatever commit this leaves.
6002    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6003        use crate::recovery::{Mode, Repair};
6004        if !self.state.reviews.is_empty() {
6005            return Ok(());
6006        }
6007        let Some(mut rec) = self.state.recovery.clone() else {
6008            return Ok(());
6009        };
6010        if rec.mode != Mode::VerifyRepair {
6011            return Ok(());
6012        }
6013        let wt = winner.worktree.clone();
6014        let mut head = git::rev_parse(&wt, "HEAD").await?;
6015        match rec.repair.clone() {
6016            Repair::Done { .. } => return Ok(()),
6017            Repair::WaitingForRestore { before } => {
6018                self.graft_restore(&wt, &before).await?;
6019                head = before;
6020                rec.repair = Repair::NotStarted;
6021                self.state.recovery = Some(rec.clone());
6022                self.state.save()?;
6023            }
6024            Repair::Started { before } => {
6025                // Interrupted: the unfinished edits are not trusted.
6026                // An unconfirmed rollback blocks the run with `Started` kept,
6027                // so a resume retries it; the tree is never reviewed as is.
6028                if let Err(e) = self.graft_restore(&wt, &before).await {
6029                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6030                        return Err(e);
6031                    }
6032                    return self
6033                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6034                        .await;
6035                }
6036                rec.repair = Repair::Done {
6037                    before: before.clone(),
6038                    after: before,
6039                    committed: false,
6040                    failed: Some("interrupted; rolled back".to_owned()),
6041                };
6042                self.state
6043                    .event("recovery", "the repair was interrupted and rolled back");
6044                self.state.recovery = Some(rec);
6045                self.state.save()?;
6046                return Ok(());
6047            }
6048            Repair::NotStarted => {}
6049        }
6050        if crate::recovery::recheck(&mut rec, &head) {
6051            self.state.event("recovery", rec.summary());
6052            self.state.recovery = Some(rec);
6053            self.state.save()?;
6054            return Ok(());
6055        }
6056        rec.repair = Repair::Started {
6057            before: head.clone(),
6058        };
6059        self.state.recovery = Some(rec.clone());
6060        self.state.save()?;
6061
6062        let language = self.state.config.graph.language.clone();
6063        let sessions = self.state.config.graph.sessions;
6064        let artifacts = agent::artifacts_dir(&self.state.dir());
6065        let fix_prompt = prompt::recovery_fix(
6066            &self.state.instruction,
6067            &rec,
6068            self.state.config.graph.review_rounds,
6069            &language,
6070        );
6071        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6072        self.state.event(
6073            "recovery",
6074            format!(
6075                "repair: the fixer starts from the failed verification on {}",
6076                short(&head)
6077            ),
6078        );
6079        let (_job, seat, out) = self
6080            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6081                prompt: fix_prompt.clone(),
6082                spec,
6083                seat,
6084                cwd: wt.clone(),
6085                timeout,
6086                allow_write: true,
6087                sessions,
6088                artifacts: artifacts.clone(),
6089                stem: "fix-recovery".to_owned(),
6090                handover: None,
6091            })
6092            .await;
6093        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6094            rec.repair = Repair::WaitingForRestore { before: head };
6095            self.state.recovery = Some(rec);
6096            return self.stop_for_resource();
6097        }
6098        let failed = match &out {
6099            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6100            AgentOutcome::Quota(o) => {
6101                self.state.quota.push(QuotaLoss {
6102                    seat: seat.key.clone(),
6103                    node: "fix".to_owned(),
6104                    at: Timestamp::now(),
6105                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6106                });
6107                Some("rate limited (quota); the fixer could not run".to_owned())
6108            }
6109            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6110            AgentOutcome::Failed(e) => Some(e.clone()),
6111        };
6112        self.state.seats.insert(seat.key.clone(), seat);
6113        let coordination = self.coordinate(&wt).await?;
6114        let failed = match failed {
6115            // The agent's partial edits are not kept.
6116            Some(why) => {
6117                if let Err(e) = git::restore_to(&wt, &head).await {
6118                    return self
6119                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6120                        .await;
6121                }
6122                Some(why)
6123            }
6124            None => {
6125                if let Ok(r) = git::rescue_commit(
6126                    &wt,
6127                    "magi: recovery repair of the failed verification (uncommitted work)",
6128                )
6129                .await
6130                {
6131                    self.state.note_withheld("fix", &r.withheld);
6132                }
6133                None
6134            }
6135        };
6136        let after = git::rev_parse(&wt, "HEAD").await?;
6137        drop(coordination);
6138        self.state.event(
6139            "recovery",
6140            match &failed {
6141                Some(why) => format!("repair: nothing kept ({why})"),
6142                None if after != head => {
6143                    format!("repair: committed {}", short(&after))
6144                }
6145                None => "repair: the fixer made no commit".to_owned(),
6146            },
6147        );
6148        rec.repair = Repair::Done {
6149            before: head.clone(),
6150            committed: after != head,
6151            after,
6152            failed,
6153        };
6154        self.state.recovery = Some(rec);
6155        self.state.save()?;
6156        Ok(())
6157    }
6158
6159    /// A repair whose rollback could not be confirmed: the run is blocked and
6160    /// resumable, and the unconfirmed state is never reviewed.
6161    async fn stop_recovery(
6162        &mut self,
6163        mut rec: crate::recovery::RecoveryRecord,
6164        head: &str,
6165        why: String,
6166    ) -> Result<()> {
6167        // `Started` stays recorded: a resume retries the restore.
6168        rec.repair = crate::recovery::Repair::Started {
6169            before: head.to_owned(),
6170        };
6171        self.state.recovery = Some(rec);
6172        self.state
6173            .event("recovery", format!("repair could not be undone: {why}"));
6174        self.state.status = RunStatus::Blocked;
6175        self.state.save()?;
6176        bail!("recovery repair could not be rolled back: {why}")
6177    }
6178
6179    async fn review_loop(&mut self) -> Result<()> {
6180        if self.hold_base_sync().await? {
6181            return Ok(());
6182        }
6183        loop {
6184            if self
6185                .state
6186                .base_sync
6187                .as_ref()
6188                .is_some_and(|s| s.conflict.is_some())
6189            {
6190                return Ok(());
6191            }
6192            if self.catchup_repairable() {
6193                let record = self.state.reviews.last().expect("repairable round").clone();
6194                let winner = self
6195                    .state
6196                    .winner()
6197                    .cloned()
6198                    .context("catch-up repair has no winner")?;
6199                if !self.fix_review_round(&winner, record).await? {
6200                    return Ok(());
6201                }
6202            }
6203            self.review_loop_pass().await?;
6204            if !self.catchup_repairable() {
6205                return Ok(());
6206            }
6207        }
6208    }
6209
6210    /// Derive an owed catch-up repair from durable round evidence, including
6211    /// reentry after a crash before the fixer was dispatched.
6212    fn catchup_repairable(&self) -> bool {
6213        self.state.reviews.len() < self.state.config.graph.review_rounds
6214            && self.state.reviews.last().is_some_and(|r| {
6215                !r.clean
6216                    && r.blocking == 0
6217                    && r.fix.is_none()
6218                    && r.e2e_status() == E2eStatus::Failed
6219            })
6220    }
6221
6222    async fn review_loop_pass(&mut self) -> Result<()> {
6223        // A base that would not rebase is a person's decision, not a review
6224        // round: nothing here would change the answer, and reviewers and a
6225        // fixer would be spending real budget on a tree that cannot land
6226        // regardless of what they find.
6227        if self
6228            .state
6229            .base_sync
6230            .as_ref()
6231            .is_some_and(|s| s.conflict.is_some())
6232        {
6233            return Ok(());
6234        }
6235        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
6236        // agent files with `magi task add` name the run that paid for it. The
6237        // prompt overlay is cloned alongside it because the waves borrow it
6238        // while `self` is mutably borrowed by the node's own bookkeeping.
6239        let run_id = self.state.id.clone();
6240        let prompts = self.state.config.prompts.clone();
6241        let Some(winner) = self.state.winner().cloned() else {
6242            return Ok(());
6243        };
6244        let max_rounds = self.state.config.graph.review_rounds;
6245        // A clean round, an exhausted round budget, or a stalled tree (see
6246        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
6247        // they are recorded — recomputed here, not read off `status`, so a
6248        // reentry into a run that already stopped restates the identical
6249        // verdict instead of silently handing back whatever an earlier node
6250        // in this same walk clobbered `status` to (a solo-candidate
6251        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
6252        // below runs an empty range once the budget is spent, and would
6253        // otherwise fall through without touching `status` at all.
6254        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
6255            // A reentry after a crash between the last round's save and
6256            // `stop_reviewing` reaches the hand-off here, not there.
6257            if status == RunStatus::Gating {
6258                self.record_contested_handoff();
6259            }
6260            self.state.status = status;
6261            self.state.save()?;
6262            return Ok(());
6263        }
6264        self.state.status = RunStatus::Reviewing;
6265        // A last recorded round whose own verification never resolved
6266        // (`ResourceBlocked` — the shared build cache, not the patch) is
6267        // never a concluded round, whatever the round budget says: starting
6268        // a fresh round on top of it would spend a whole new reviewer wave
6269        // re-reading an unchanged patch instead of just retrying the one
6270        // check that actually needs it, and once the budget is spent the
6271        // loop below has nothing left to do at all (its range is empty).
6272        // Retry that check directly instead, exactly the same retry
6273        // `stop_reviewing` already does for its own catch-up case.
6274        if self.state.reviews.last().is_some_and(|r| {
6275            r.e2e_status() == E2eStatus::ResourceBlocked
6276                || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6277        }) {
6278            let shell = self.state.config.shell();
6279            return self
6280                .stop_unresolved_review(
6281                    "the last round's own verification never resolved",
6282                    &shell,
6283                    &winner.worktree,
6284                )
6285                .await;
6286        }
6287
6288        self.recovery_repair(&winner).await?;
6289        if self.resource_halt()? {
6290            return Ok(());
6291        }
6292
6293        let repo = self.state.repo.clone();
6294        let root = self.state.worktree_root();
6295        let language = self.state.config.graph.language.clone();
6296        let sessions = self.state.config.graph.sessions;
6297        let artifacts = self.merge_repair_artifacts();
6298        let base = self.landing_base();
6299        let base_short = short(&base);
6300        let reviewers = self.roles.reviewers.clone();
6301        let shell = self.state.config.shell();
6302
6303        for round in (self.state.reviews.len() + 1)..=max_rounds {
6304            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6305            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6306            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6307            // The prior round's own record, already persisted — never a
6308            // hand-carried variable of just its failing output: that is
6309            // exactly what let a round's e2e result drift out of sync with
6310            // which commit it was actually about (see `SCHEMA`'s doc for
6311            // schema 8). Judged against `head`, the commit reviewers are
6312            // about to look at now, so the summary always reads as "an
6313            // earlier head" here — this round's own patch has not been
6314            // checked yet.
6315            let prev_verification = self
6316                .state
6317                .reviews
6318                .last()
6319                .and_then(|r| r.verification_summary(&head));
6320            // Only the first round of a run that took a branch over: later
6321            // rounds have this run's own history.
6322            let prior_section = match (&self.state.recovery, round) {
6323                (Some(rec), 1) if self.state.reviews.is_empty() => {
6324                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6325                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6326                            Some(crate::recovery::head_of(
6327                                &git::diff(&winner.worktree, reviewed, "HEAD")
6328                                    .await
6329                                    .unwrap_or_default(),
6330                                crate::recovery::SINCE_DIFF_MAX,
6331                            ))
6332                        }
6333                        _ => None,
6334                    };
6335                    Some(prompt::recovery_review(rec, since.as_deref()))
6336                }
6337                _ => None,
6338            };
6339
6340            // Each reviewer gets its own detached checkout of exactly this
6341            // commit: nobody can perturb the winner's tree, and the fixer can
6342            // keep working without racing a reviewer.
6343            let mut jobs = Vec::new();
6344            let coordination = self.coordinate(&repo).await?;
6345            for (r, spec) in reviewers.iter().cloned().enumerate() {
6346                let wt = root.join(format!("review-{}", r + 1));
6347                // `clean -fdx` in a reset wipes setup's products, so every
6348                // round sets the seat up again.
6349                if wt.exists() {
6350                    git::reset_detached(&wt, &head).await?;
6351                } else {
6352                    git::worktree_add_detached(&repo, &wt, &head).await?;
6353                }
6354                self.setup_seat_worktree(&repo, &wt, false).await?;
6355                let seat_key = format!("review-{}", r + 1);
6356                // The seat starts the round on whoever answered it last, not
6357                // on the agent the spec names, so a failure is not re-paid.
6358                let spec = pick_start_spec(
6359                    &self.roles.reviewer_roster,
6360                    spec,
6361                    self.state.seat_history.get(&seat_key),
6362                );
6363                let seat = self.seat(&seat_key, &spec.id);
6364                jobs.push(SeatJob {
6365                    prompt: prompt::review(&prompt::ReviewCtx {
6366                        instruction: &self.state.instruction,
6367                        branch: &winner.branch,
6368                        base_short: &base_short,
6369                        stat: &stat,
6370                        patch: &patch,
6371                        verification: prev_verification.as_ref(),
6372                        prior: prior_section.as_deref(),
6373                        reviewers: reviewers.len(),
6374                        round,
6375                        rounds: max_rounds,
6376                        // A review-only run has no rankings, so nothing
6377                        // competed for this patch and the reviewer is told so.
6378                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6379                        lens: Lens::for_seat(r),
6380                        language: &language,
6381                        github_language: &self.state.config.graph.github_language,
6382                    }),
6383                    spec,
6384                    seat,
6385                    cwd: wt,
6386                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6387                    allow_write: false,
6388                    sessions,
6389                    artifacts: artifacts.clone(),
6390                    stem: format!("review-{round}-{}", r + 1),
6391                    handover: None,
6392                });
6393            }
6394
6395            drop(coordination);
6396            self.state.event(
6397                "review",
6398                format!(
6399                    "round {round}: {} reviewers on {}",
6400                    jobs.len(),
6401                    short(&head)
6402                ),
6403            );
6404            let mut quota_losses = Vec::new();
6405            let review_retries = self.state.config.graph.retries;
6406            let review_cache = self.state.config.cache_dir();
6407            let ctx = WaveCtx {
6408                carry_seats: true,
6409                pauses: Some((&self.pause, &self.interrupt)),
6410                run: &run_id,
6411                node: "review",
6412                prompts: &prompts,
6413                cache: review_cache.as_deref(),
6414                round: Some(round),
6415            };
6416            let results = ask_json_wave::<Review>(
6417                jobs,
6418                Arc::clone(&self.sem),
6419                review_retries,
6420                &self.roles.reviewer_roster,
6421                &ctx,
6422                &mut quota_losses,
6423                &mut self.state,
6424                &|_: &Review| Ok(()),
6425            )
6426            .await;
6427            if self.state.resource_block.is_some() {
6428                return self.stop_for_resource();
6429            }
6430            // Counted before the move below: how many of *this* round's
6431            // reviewer seats were lost to their own rate limit, as opposed to
6432            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6433            let round_quota_missing = quota_losses.len();
6434            self.state.quota.extend(quota_losses);
6435
6436            let mut records = Vec::new();
6437            let mut all_findings = Vec::new();
6438            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6439                let agent_id = seat.agent.clone();
6440                self.state.seats.insert(seat.key.clone(), seat);
6441                let mut record = ReviewRecord {
6442                    reviewer: r + 1,
6443                    agent: agent_id,
6444                    summary: String::new(),
6445                    findings: Vec::new(),
6446                    vote: None,
6447                    failed: None,
6448                    duration_ms: 0,
6449                    // Set for both outcomes: `failed: Some(_)` with
6450                    // `attempts > 0` is a seat every retry still lost, not a
6451                    // recovered one — only `failed: None` with `attempts > 0`
6452                    // reads as "answered after a nudge" (see this field's own
6453                    // doc).
6454                    attempts,
6455                };
6456                match res {
6457                    Ok((review, out)) => {
6458                        // Sanitized here, at the point every other piece of
6459                        // agent prose in this file is (candidate summaries,
6460                        // deliberation turns, vote reasons): a reviewer's own
6461                        // words are the one thing about it that could name
6462                        // it, and reconsideration below broadcasts this same
6463                        // summary and these same findings to every other
6464                        // seat on the panel.
6465                        record.summary =
6466                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6467                        record.vote = Some(review.vote);
6468                        record.duration_ms = out.duration_ms;
6469                        for (n, mut f) in review.findings.into_iter().enumerate() {
6470                            // ids are magi's, never the agent's: the fixer's
6471                            // adoption report is keyed by them.
6472                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6473                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6474                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6475                            // `file` is agent-supplied prose too, never
6476                            // checked against the real tree — the same
6477                            // exposure `title`/`detail` above have, just in
6478                            // a field easy to forget because it looks like a
6479                            // path rather than free text.
6480                            f.file = f
6481                                .file
6482                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6483                            all_findings.push(f.clone());
6484                            record.findings.push(f);
6485                        }
6486                        self.state.event(
6487                            "review",
6488                            format!(
6489                                "round {round}: reviewer {} voted {} with {} finding(s)",
6490                                r + 1,
6491                                review.vote.label(),
6492                                record.findings.len()
6493                            ),
6494                        );
6495                    }
6496                    Err(e) => {
6497                        record.failed = Some(e.to_string());
6498                        self.state.event(
6499                            "review",
6500                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6501                        );
6502                    }
6503                }
6504                records.push(record);
6505            }
6506
6507            // Tally the round's votes and, if they split, spend the one
6508            // round of reconsideration the split -> deliberate -> revote
6509            // shape `judge`/`vote` use for the panel, sized down to what a
6510            // read-only review round can afford: one round, and a revote
6511            // rather than an argument, because the panel already wrote its
6512            // reasoning down as findings the first time around.
6513            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6514            let vote_split =
6515                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6516            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6517            if vote_split {
6518                self.state.event(
6519                    "review",
6520                    format!(
6521                        "round {round}: votes split ({}) — one round of reconsideration",
6522                        initial_votes
6523                            .iter()
6524                            .map(|v| v.label())
6525                            .collect::<Vec<_>>()
6526                            .join(", ")
6527                    ),
6528                );
6529                // Seats read every seat's findings and votes, still numbered
6530                // and never named — the same anonymity `review` itself keeps.
6531                let panel: Vec<ReviewSeatReport<'_>> = records
6532                    .iter()
6533                    .filter_map(|r| {
6534                        r.vote.map(|vote| ReviewSeatReport {
6535                            reviewer: r.reviewer,
6536                            vote,
6537                            summary: &r.summary,
6538                            findings: &r.findings,
6539                        })
6540                    })
6541                    .collect();
6542
6543                let mut jobs = Vec::new();
6544                let mut seats_at = Vec::new();
6545                for (r, spec) in reviewers.iter().cloned().enumerate() {
6546                    // A seat with no initial vote has nothing to reconsider
6547                    // from and stays absent, the same as it stayed absent
6548                    // from `panel` above.
6549                    if records[r].vote.is_none() {
6550                        continue;
6551                    }
6552                    let wt = root.join(format!("review-{}", r + 1));
6553                    let seat_key = format!("review-{}", r + 1);
6554                    let spec = self.occupant(&seat_key, spec);
6555                    let seat = self.seat(&seat_key, &spec.id);
6556                    // A seat with no live session has already forgotten the
6557                    // initial review's prompt — restate the patch it is
6558                    // voting on, the same as `deliberate`/`vote` do for a
6559                    // judge in the same position.
6560                    // The panel already carries this seat's own review and
6561                    // vote, so restating the patch makes the prompt whole for
6562                    // a seat handed to another agent.
6563                    let build = |with_patch: bool| {
6564                        prompt::review_reconsider(&ReviewReconsiderCtx {
6565                            instruction: &self.state.instruction,
6566                            reviewer: r + 1,
6567                            lens: Lens::for_seat(r),
6568                            panel: &panel,
6569                            patch: with_patch.then_some(ReviewPatch {
6570                                branch: &winner.branch,
6571                                base_short: &base_short,
6572                                stat: &stat,
6573                                patch: &patch,
6574                            }),
6575                            round,
6576                            rounds: max_rounds,
6577                            language: &language,
6578                        })
6579                    };
6580                    let full = build(true);
6581                    let prompt = if has_context(&spec, &seat, sessions) {
6582                        build(false)
6583                    } else {
6584                        full.clone()
6585                    };
6586                    jobs.push(SeatJob {
6587                        prompt,
6588                        spec,
6589                        seat,
6590                        cwd: wt,
6591                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6592                        allow_write: false,
6593                        sessions,
6594                        artifacts: artifacts.clone(),
6595                        stem: format!("review-{round}-reconsider-{}", r + 1),
6596                        handover: Some(full),
6597                    });
6598                    seats_at.push(r);
6599                }
6600
6601                let mut recon_quota_losses = Vec::new();
6602                let recon_cache = self.state.config.cache_dir();
6603                let recon_ctx = WaveCtx {
6604                    carry_seats: true,
6605                    pauses: Some((&self.pause, &self.interrupt)),
6606                    run: &run_id,
6607                    node: "review",
6608                    prompts: &prompts,
6609                    cache: recon_cache.as_deref(),
6610                    round: Some(round),
6611                };
6612                let recon_results = ask_json_wave::<ReviewRevote>(
6613                    jobs,
6614                    Arc::clone(&self.sem),
6615                    review_retries,
6616                    &self.roles.reviewer_roster,
6617                    &recon_ctx,
6618                    &mut recon_quota_losses,
6619                    &mut self.state,
6620                    &|_: &ReviewRevote| Ok(()),
6621                )
6622                .await;
6623                if self.state.resource_block.is_some() {
6624                    return self.stop_for_resource();
6625                }
6626                self.state.quota.extend(recon_quota_losses);
6627
6628                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6629                    let agent_id = seat.agent.clone();
6630                    self.state.seats.insert(seat.key.clone(), seat);
6631                    let mut rec = ReviewRevoteRecord {
6632                        reviewer: r + 1,
6633                        agent: agent_id,
6634                        vote: None,
6635                        reason: String::new(),
6636                        failed: None,
6637                    };
6638                    match res {
6639                        Ok((rv, _)) => {
6640                            rec.vote = Some(rv.vote);
6641                            rec.reason =
6642                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6643                            self.state.event(
6644                                "review",
6645                                format!(
6646                                    "round {round}: reviewer {} revoted {}",
6647                                    r + 1,
6648                                    rv.vote.label()
6649                                ),
6650                            );
6651                        }
6652                        Err(e) => {
6653                            rec.failed = Some(e.to_string());
6654                            self.state.event(
6655                                "review",
6656                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6657                            );
6658                        }
6659                    }
6660                    reconsideration.push(rec);
6661                }
6662            } else if initial_votes.len() > 1 {
6663                self.state.event(
6664                    "review",
6665                    format!(
6666                        "round {round}: votes agreed ({}) — no reconsideration",
6667                        initial_votes[0].label()
6668                    ),
6669                );
6670            }
6671
6672            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6673            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6674            // A round that already has a blocking finding and a round left to
6675            // try is going back to the fixer no matter what `verify.e2e`
6676            // says, so running it first only spends the loop's slowest step
6677            // (minutes, for a Rust repo's full test suite) on a head about
6678            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6679            // runs once a round has no blocking findings left (see
6680            // `round_is_clean`, which a deferred — empty — `e2e` can never
6681            // satisfy since `blocking` is nonzero whenever this branch is
6682            // taken), and `stop_reviewing` forces a real run before it will
6683            // ever read a deferred round as green.
6684            let defer_e2e =
6685                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6686            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6687                let reason =
6688                    format!("{blocking} blocking finding(s) already required a fix this round");
6689                self.state.event(
6690                    "verify",
6691                    format!(
6692                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6693                         {}); it will run once a round has none left",
6694                        short(&head)
6695                    ),
6696                );
6697                (Vec::new(), false, true, Some(reason))
6698            } else {
6699                let e2e_commands = self.state.config.verify.e2e.clone();
6700                let cache_dir = self.state.config.cache_dir();
6701                let context = format!("round {round}");
6702                let wait = self.repository_wait();
6703                let (e2e, verify_retried) = with_cache_lease(
6704                    &mut self.state,
6705                    cache_dir.as_deref(),
6706                    "e2e",
6707                    "e2e",
6708                    &winner.worktree,
6709                    &head,
6710                    verify_timeout,
6711                    &context,
6712                    &wait,
6713                    |state, budget| {
6714                        let shell = shell.clone();
6715                        let e2e_commands = e2e_commands.clone();
6716                        let worktree = winner.worktree.clone();
6717                        let context = context.clone();
6718                        async move {
6719                            run_e2e_with_retry(
6720                                state,
6721                                &shell,
6722                                &e2e_commands,
6723                                &worktree,
6724                                budget,
6725                                &context,
6726                            )
6727                            .await
6728                        }
6729                    },
6730                )
6731                .await?;
6732                (e2e, verify_retried, false, None)
6733            };
6734
6735            let expected = records.len();
6736            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6737            let incomplete = answered < expected;
6738            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6739            let policy = self.state.config.graph.incomplete_review;
6740            let clean = round_is_clean(
6741                blocking,
6742                e2e_ok,
6743                answered,
6744                expected,
6745                round_quota_missing,
6746                policy,
6747            );
6748
6749            let mut round_record = ReviewRound {
6750                round,
6751                head: head.clone(),
6752                verified_head: None,
6753                verified_at: None,
6754                reviews: records,
6755                e2e,
6756                verify_retried,
6757                e2e_deferred,
6758                e2e_defer_reason,
6759                fix: None,
6760                blocking,
6761                answered,
6762                expected,
6763                clean,
6764                progressed: false,
6765                vote_split,
6766                reconsideration,
6767                verdict: None,
6768            };
6769            // The final vote per seat is its revote where reconsideration
6770            // ran and answered, its initial vote otherwise — the same
6771            // fallback `tally` uses for a judge whose private vote failed.
6772            round_record.verdict = ReviewVote::worst(
6773                round_record
6774                    .final_votes()
6775                    .into_iter()
6776                    .map(|(_, _, vote)| vote),
6777            );
6778            // Which commit and when magi actually attempted to check —
6779            // known the moment a command was dispatched against `head`,
6780            // whether or not it finished: a resource-blocked attempt still
6781            // targeted a specific commit at a specific time, and leaving
6782            // that unrecorded is exactly what made `verification_summary`
6783            // report a fresh attempt as "commit unknown ... recorded before
6784            // this was tracked", indistinguishable from a genuinely old,
6785            // untracked record. Only a deferred or unconfigured round never
6786            // ran at all and has nothing to record — see
6787            // `ReviewRound::verified_head`'s own doc.
6788            if !matches!(
6789                round_record.e2e_status(),
6790                E2eStatus::Deferred | E2eStatus::NotConfigured
6791            ) {
6792                round_record.verified_head = Some(head.clone());
6793                round_record.verified_at = Some(Timestamp::now());
6794            }
6795
6796            if incomplete {
6797                let missing: Vec<String> = round_record
6798                    .reviews
6799                    .iter()
6800                    .filter(|r| r.failed.is_some())
6801                    .map(|r| format!("review-{}", r.reviewer))
6802                    .collect();
6803                self.state.event(
6804                    "review",
6805                    format!(
6806                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6807                        missing.join(", ")
6808                    ),
6809                );
6810            }
6811
6812            if clean {
6813                self.state.event(
6814                    "review",
6815                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6816                        format!(
6817                            "round {round}: clean (warn policy, incomplete panel) — no \
6818                             blocking findings from the seats that answered, verification green"
6819                        )
6820                    } else if incomplete {
6821                        format!(
6822                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6823                             quorum) — no blocking findings from the seats that answered, \
6824                             verification green",
6825                            expected - answered
6826                        )
6827                    } else {
6828                        format!("round {round}: clean — no blocking findings, verification green")
6829                    },
6830                );
6831                self.state.reviews.push(round_record);
6832                self.state.status = RunStatus::Gating;
6833                self.state.save()?;
6834                return Ok(());
6835            }
6836
6837            // Nothing was raised and verification passed, but not every seat
6838            // answered and `round_is_clean` still refused to call it clean —
6839            // either a seat is missing for a reason other than its own quota
6840            // (a crash, a timeout, unparsable output — worth another try), or
6841            // every seat that could have answered lost its quota and nobody
6842            // is left to decide on: re-review rather than send the fixer
6843            // after a round with nothing to fix.
6844            if incomplete && blocking == 0 && e2e_ok {
6845                self.state.reviews.push(round_record);
6846                self.state.save()?;
6847                if round == max_rounds {
6848                    self.state.status = RunStatus::Blocked;
6849                    self.state.event(
6850                        "review",
6851                        format!(
6852                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6853                             refusing to call it clean",
6854                            expected - answered
6855                        ),
6856                    );
6857                    return Ok(());
6858                }
6859                continue;
6860            }
6861
6862            // Nothing for the fixer to act on (`blocking == 0`) and the only
6863            // reason this round is not clean is that magi itself never got
6864            // a command to run — the shared build cache, not the patch (see
6865            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6866            // the fixer would invite a change to appease contention that has
6867            // nothing to do with the diff, and would leave this attempt
6868            // sitting in the next round's prompt as if it were about an
6869            // earlier, superseded commit rather than what it actually is:
6870            // the same head, still waiting to be checked. Wait for it the
6871            // same way the final round's own contention is already handled,
6872            // whatever round this happens to be.
6873            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6874                self.state.reviews.push(round_record);
6875                return self
6876                    .stop_unresolved_review(
6877                        "the round's own verification could not run",
6878                        &shell,
6879                        &winner.worktree,
6880                    )
6881                    .await;
6882            }
6883
6884            if round == max_rounds {
6885                self.state.reviews.push(round_record);
6886                return self
6887                    .stop_reviewing(
6888                        &format!(
6889                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6890                        ),
6891                        &shell,
6892                        &winner.worktree,
6893                    )
6894                    .await;
6895            }
6896
6897            if !self.fix_review_round(&winner, round_record).await? {
6898                return Ok(());
6899            }
6900        }
6901        Ok(())
6902    }
6903
6904    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
6905    /// False means admission or continuation policy stopped the review loop.
6906    async fn fix_review_round(
6907        &mut self,
6908        winner: &Candidate,
6909        mut round_record: ReviewRound,
6910    ) -> Result<bool> {
6911        let round = round_record.round;
6912        let shell = self.state.config.shell();
6913        let step = self.run_fix_round(winner, &mut round_record).await?;
6914        if matches!(step, FixStep::Resource) {
6915            self.stop_for_resource()?;
6916            return Ok(false);
6917        }
6918        if self.state.reviews.len() < round {
6919            self.state.reviews.push(round_record);
6920        } else {
6921            self.state.reviews[round - 1] = round_record;
6922        }
6923        self.state.save()?;
6924
6925        // The fixer's own report never came back this round, even after
6926        // `continue_fix_report`'s own budget was spent on it — not an
6927        // ordinary "no report" (dropped stream, quota, plain failure),
6928        // which already reads that way and is left to the existing round
6929        // budget. Stopping here, rather than opening another round, is
6930        // what keeps a next reviewer/fixer wave from ever being
6931        // dispatched onto `winner.worktree` while whatever the seat's
6932        // last call may still have running there is unaccounted for: no
6933        // process liveness check exists (and none is being added — see
6934        // AGENTS.md/this task's own scope), so the only way to honour
6935        // "nothing starts before a valid report returns" is to not start
6936        // anything further on this worktree from this run at all.
6937        if matches!(step, FixStep::Recorded { report_lost: true }) {
6938            self.stop_reviewing(
6939                "the fixer's adoption report never came back, even after resuming its \
6940                         own seat; refusing to start another round against the same worktree \
6941                         while that is unresolved",
6942                &shell,
6943                &winner.worktree,
6944            )
6945            .await?;
6946            return Ok(false);
6947        }
6948
6949        let streak = self
6950            .state
6951            .reviews
6952            .iter()
6953            .rev()
6954            .take_while(|r| !r.progressed)
6955            .count();
6956        if streak >= STAGNANT_LIMIT {
6957            self.stop_reviewing(
6958                &format!("the tree has not moved against base for {streak} round(s) in a row"),
6959                &shell,
6960                &winner.worktree,
6961            )
6962            .await?;
6963            return Ok(false);
6964        }
6965        Ok(true)
6966    }
6967
6968    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
6969    async fn run_fix_round(
6970        &mut self,
6971        winner: &Candidate,
6972        round_record: &mut ReviewRound,
6973    ) -> Result<FixStep> {
6974        let round = round_record.round;
6975        let max_rounds = self.state.config.graph.review_rounds;
6976        let run_id = self.state.id.clone();
6977        let prompts = self.state.config.prompts.clone();
6978        let language = self.state.config.graph.language.clone();
6979        let sessions = self.state.config.graph.sessions;
6980        let artifacts = self.merge_repair_artifacts();
6981        let base = self.landing_base();
6982        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6983        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6984        let this_round_verification = round_record.verification_summary(&head);
6985        let all_findings: Vec<_> = round_record
6986            .reviews
6987            .iter()
6988            .flat_map(|r| r.findings.iter().cloned())
6989            .collect();
6990        // Fix. The winner's own implementer seat continues its conversation:
6991        // the competition is over, so context is pure benefit now.
6992        let blocking_findings: Vec<_> = all_findings
6993            .iter()
6994            .filter(|f| f.severity.blocks())
6995            .cloned()
6996            .collect();
6997        let fix_prompt = prompt::fix_in(
6998            &self.state.instruction,
6999            &blocking_findings,
7000            this_round_verification.as_ref(),
7001            round,
7002            max_rounds,
7003            &language,
7004            &self.state.config.graph.github_language,
7005        );
7006        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7007        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7008        let (job, seat, out) = self
7009            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7010                prompt: fix_prompt.clone(),
7011                spec,
7012                seat,
7013                cwd: winner.worktree.clone(),
7014                timeout,
7015                allow_write: true,
7016                sessions,
7017                artifacts: artifacts.clone(),
7018                stem: format!("fix-{round}"),
7019                handover: None,
7020            })
7021            .await;
7022        // Not asked: the round is not recorded (so a resume repeats it),
7023        // no fixer was handed over to, and the seat is left as it was.
7024        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7025            return Ok(FixStep::Resource);
7026        }
7027        let agent_id = seat.agent.clone();
7028
7029        let mut fix = FixRecord {
7030            agent: agent_id,
7031            addressed: Vec::new(),
7032            rejected: Vec::new(),
7033            notes: String::new(),
7034            committed: false,
7035            failed: None,
7036            duration_ms: 0,
7037            continuation: None,
7038        };
7039        let mut continuation = ContinuationRecord::not_needed();
7040        let mut final_seat = seat.clone();
7041        match out {
7042            AgentOutcome::Ok(o) => {
7043                fix.duration_ms = o.duration_ms;
7044                let parsed = verdict::extract_json::<FixReport>(&o.text);
7045                // A parsed report standing next to a command this same
7046                // reply's own CLI never confirmed the exit status of is
7047                // not a resolved answer — the identical `CommandEvidence`
7048                // `state.jobs` renders, read here instead of only on
7049                // display, per the completion judgment and the shown
7050                // record needing to agree.
7051                let incomplete_reason = match &parsed {
7052                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7053                        "the reply parsed, but it reported a command whose own CLI never \
7054                             confirmed an exit status"
7055                            .to_owned(),
7056                    ),
7057                    Ok(_) => None,
7058                    Err(e) => Some(e.to_string()),
7059                };
7060                match incomplete_reason {
7061                    None => {
7062                        let report = parsed.expect("checked Ok above");
7063                        fix.addressed = report.addressed;
7064                        fix.rejected = report.rejected;
7065                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7066                    }
7067                    Some(reason) => {
7068                        let (resumed_seat, resolved, failure, cont) = self
7069                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7070                            .await;
7071                        fix.duration_ms += cont.cumulative_wait_ms;
7072                        continuation = cont;
7073                        final_seat = resumed_seat;
7074                        match resolved {
7075                            Some(report) => {
7076                                fix.addressed = report.addressed;
7077                                fix.rejected = report.rejected;
7078                                fix.notes =
7079                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
7080                            }
7081                            None => fix.failed = failure,
7082                        }
7083                    }
7084                }
7085            }
7086            // The CLI's raw error JSON is not a fix report to parse.
7087            AgentOutcome::Dropped(o) => {
7088                fix.duration_ms = o.duration_ms;
7089                let why = o
7090                    .dropped
7091                    .as_ref()
7092                    .map(|d| d.why.as_str())
7093                    .unwrap_or("the CLI ended the stream without delivering its answer");
7094                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7095            }
7096            AgentOutcome::Quota(o) => {
7097                self.state.quota.push(QuotaLoss {
7098                    seat: final_seat.key.clone(),
7099                    node: "fix".to_owned(),
7100                    at: Timestamp::now(),
7101                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7102                });
7103                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7104            }
7105            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7106        }
7107        fix.continuation = Some(continuation);
7108        self.state.seats.insert(final_seat.key.clone(), final_seat);
7109        let coordination = self.coordinate(&winner.worktree).await?;
7110        if let Ok(r) = git::rescue_commit(
7111            &winner.worktree,
7112            &format!("magi: review round {round} fixes (uncommitted work)"),
7113        )
7114        .await
7115        {
7116            self.state.note_withheld("fix", &r.withheld);
7117        }
7118        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7119        fix.committed = after != before;
7120        // Judged by what `git` says moved against base, never by the
7121        // fixer's own `addressed`/`rejected` count — see
7122        // `ReviewRound::progressed`. Propagated with `?`, the same as the
7123        // `patch` snapshot above: swallowing this error would default
7124        // `diff_after` to empty, which almost always differs from a
7125        // non-empty `patch` and reads as "progressed" — exactly backwards
7126        // for a `git` failure the stagnation check cannot see through.
7127        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7128        drop(coordination);
7129        let progressed = diff_after != patch;
7130        let commit_note = if fix.committed {
7131            "committed"
7132        } else {
7133            "NO new commit"
7134        };
7135        let tree_note = if progressed {
7136            "changed vs base"
7137        } else {
7138            "unchanged vs base"
7139        };
7140        self.state.event(
7141            "fix",
7142            match &fix.failed {
7143                // Distinct on purpose from "0 addressed, 0 rejected": the
7144                // fixer's own diff still landed (blocking counts do keep
7145                // falling round over round), only its adoption report did
7146                // not come back, so this must never read like every
7147                // finding was reviewed and declined.
7148                Some(reason) => {
7149                    format!(
7150                        "round {round}: fixer's adoption report was lost ({reason}); \
7151                             {commit_note}, tree {tree_note}"
7152                    )
7153                }
7154                None => format!(
7155                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7156                         {tree_note}{}",
7157                    fix.addressed.len(),
7158                    fix.rejected.len(),
7159                    if continuation.outcome == ContinuationOutcome::Resumed {
7160                        format!(
7161                            " (adoption report recovered after {} continuation(s))",
7162                            continuation.attempts
7163                        )
7164                    } else {
7165                        String::new()
7166                    },
7167                ),
7168            },
7169        );
7170        round_record.fix = Some(fix);
7171        round_record.progressed = progressed;
7172        Ok(FixStep::Recorded {
7173            report_lost: matches!(
7174                continuation.outcome,
7175                ContinuationOutcome::Exhausted
7176                    | ContinuationOutcome::QuotaLost
7177                    | ContinuationOutcome::NoSession
7178            ),
7179        })
7180    }
7181
7182    /// Decide, from the last recorded round's own verification, whether
7183    /// stopping the review loop is a hand-off or a genuine block.
7184    ///
7185    /// Called once the loop has given up trying — the round budget is spent,
7186    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
7187    /// findings still open, never while a round is still clean or the
7188    /// incomplete-panel case handled inline above. Gate and e2e are facts
7189    /// about the tree; a lingering review finding is an opinion, and this
7190    /// workload's own `magi stats` puts reviewer precision low enough
7191    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
7192    /// must not by itself stand between a green, verified change and the
7193    /// human who decides what to do with it. A red e2e is not an opinion, so
7194    /// that case still blocks, with the failing command and a tail of its
7195    /// output recorded here rather than left in `run.json` for someone to go
7196    /// find.
7197    ///
7198    /// A round that deferred its own e2e (see [`Config::graph`]'s
7199    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
7200    /// only because nothing ran, and treating an empty list as a passing one
7201    /// here is exactly the "deferred painted green" bug this function exists
7202    /// to not have. When the last round's own verification never resolved —
7203    /// deferred on purpose, or a real attempt the shared build cache blocked
7204    /// — this makes (or retries) the real run, on the actual worktree this
7205    /// loop is about to stop touching, before deciding anything. A
7206    /// resource-blocked attempt is likewise never read as either green or
7207    /// red: it is evidence about the machine, not the patch (see
7208    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
7209    /// blocked cache leaves this call without deciding rather than guessing
7210    /// — the caller retries on a later reentry.
7211    /// Record, once, that the review loop handed off over a blocking finding
7212    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
7213    /// `land` asks the owner even with `land_approval` off. Called from every
7214    /// path that concludes `Gating`; a reentry keeps the first record.
7215    fn record_contested_handoff(&mut self) {
7216        if self.state.contested_handoff.is_some() {
7217            return;
7218        }
7219        let Some(contested) = self
7220            .state
7221            .reviews
7222            .last()
7223            .and_then(ReviewRound::contested_handoff)
7224        else {
7225            return;
7226        };
7227        self.state.event(
7228            "review",
7229            format!(
7230                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7231                 wait for the owner's approval",
7232                contested.findings.len(),
7233                contested.rejecters.len()
7234            ),
7235        );
7236        self.state.contested_handoff = Some(contested);
7237    }
7238
7239    async fn stop_unresolved_review(
7240        &mut self,
7241        why: &str,
7242        shell: &[String],
7243        worktree: &Path,
7244    ) -> Result<()> {
7245        self.stop_reviewing_inner(why, shell, worktree, true).await
7246    }
7247
7248    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7249        self.stop_reviewing_inner(why, shell, worktree, false).await
7250    }
7251
7252    async fn stop_reviewing_inner(
7253        &mut self,
7254        why: &str,
7255        shell: &[String],
7256        worktree: &Path,
7257        repairable: bool,
7258    ) -> Result<()> {
7259        let round_idx = self.state.reviews.len() - 1;
7260        // A deferred round and a resource-blocked one are the same shape
7261        // here: neither has a real result yet, and both get one more
7262        // attempt. Read off `e2e_status` — the single source for this —
7263        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
7264        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
7265        // still retries instead of being read as a settled result the
7266        // instant it stops being empty.
7267        let needs_catchup_run = matches!(
7268            self.state.reviews[round_idx].e2e_status(),
7269            E2eStatus::Deferred | E2eStatus::ResourceBlocked
7270        );
7271        if needs_catchup_run {
7272            let round = self.state.reviews[round_idx].round;
7273            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7274            let commands = self.state.config.verify.e2e.clone();
7275            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7276            let cache_dir = self.state.config.cache_dir();
7277            let context = format!(
7278                "round {round}: verification unresolved, catching up before the final decision"
7279            );
7280            let wait = self.repository_wait();
7281            let (outcomes, verify_retried) = with_cache_lease(
7282                &mut self.state,
7283                cache_dir.as_deref(),
7284                "e2e",
7285                "e2e",
7286                worktree,
7287                &attempted_head,
7288                timeout,
7289                &context,
7290                &wait,
7291                |state, budget| {
7292                    let shell = shell.to_vec();
7293                    let commands = commands.clone();
7294                    let context = context.clone();
7295                    async move {
7296                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7297                            .await
7298                    }
7299                },
7300            )
7301            .await?;
7302            let last = &mut self.state.reviews[round_idx];
7303            last.e2e = outcomes;
7304            last.verify_retried = verify_retried;
7305            // Always the commit and time this attempt actually targeted,
7306            // whether or not it happens to equal the reviewed `head` and
7307            // whether or not a command finished — see
7308            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
7309            // attempt is recorded too, so a later reader sees "attempted
7310            // again at T2" rather than silence.
7311            last.verified_head = Some(attempted_head);
7312            last.verified_at = Some(Timestamp::now());
7313            if verify_inconclusive(&last.e2e) {
7314                // Still not a real result: `e2e_deferred` is left exactly
7315                // as it was, so `needs_catchup_run` above reads
7316                // `ResourceBlocked` (via `e2e_status`, which checks
7317                // `resource_blocked` before `e2e_deferred`) and retries
7318                // again on the next reentry, rather than recording
7319                // contention as a red e2e and blocking the run on it.
7320                self.state.save()?;
7321                return Ok(());
7322            }
7323            last.e2e_deferred = false;
7324        }
7325        let last = &self.state.reviews[round_idx];
7326        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7327
7328        match last.e2e_status() {
7329            E2eStatus::Failed => {
7330                let red: Vec<String> = last
7331                    .e2e
7332                    .iter()
7333                    .filter(|o| !o.ok())
7334                    .map(|o| {
7335                        format!(
7336                            "`{}` -> {:?}\n{}",
7337                            o.command,
7338                            o.code,
7339                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7340                        )
7341                    })
7342                    .collect();
7343                let can_repair = last.blocking == 0 && last.fix.is_none();
7344                self.state.reviews[round_idx].clean = false;
7345                self.state
7346                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7347                self.state.status = RunStatus::Blocked;
7348                if repairable
7349                    && can_repair
7350                    && self.state.reviews.len() < self.state.config.graph.review_rounds
7351                {
7352                    self.state.status = RunStatus::Reviewing;
7353                }
7354            }
7355            // `needs_catchup_run` above already retried once this call; if
7356            // it is still blocked, this is magi's own admission it could
7357            // not get a command to run, never a verdict on the patch — the
7358            // run is left exactly where a later reentry can retry again.
7359            E2eStatus::ResourceBlocked => {
7360                self.state.event(
7361                    "review",
7362                    format!(
7363                        "{why}; e2e could not run (resource unavailable); not \
7364                         deciding yet"
7365                    ),
7366                );
7367            }
7368            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7369                self.state.event(
7370                    "review",
7371                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7372                );
7373                self.record_contested_handoff();
7374                self.state.status = RunStatus::Gating;
7375            }
7376        }
7377        self.state.save()?;
7378        Ok(())
7379    }
7380
7381    // ----------------------------------------------------------------- gate
7382
7383    async fn gate(&mut self) -> Result<()> {
7384        if self.hold_base_sync().await? {
7385            return Ok(());
7386        }
7387        // Judged by the review record itself, not by `status`: a solo
7388        // candidate's `judge`/`deliberate` skip rewrites `status` on every
7389        // reentry (see `judge`), and trusting it here is exactly how a run
7390        // that exhausted its review budget got gated and merged a second
7391        // time around. `review_conclusion` recomputes the review loop's own
7392        // verdict from the round records themselves — `Gating` for a clean
7393        // round or a hand-off (see `stop_reviewing`), anything else means the
7394        // loop is still going or genuinely blocked.
7395        // A base the winner could not be replayed onto is a decision, not a
7396        // round: there is no landing tree to gate. Read as its own record for
7397        // the same reason the review verdict is.
7398        if self.state.status == RunStatus::Failed
7399            || self
7400                .state
7401                .base_sync
7402                .as_ref()
7403                .is_some_and(|s| s.conflict.is_some())
7404            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7405                != Some(RunStatus::Gating)
7406        {
7407            return Ok(());
7408        }
7409        let Some(winner) = self.state.winner().cloned() else {
7410            return Ok(());
7411        };
7412        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7413        self.state.reconcile_verification_head(&head);
7414        self.state.save()?;
7415        let wait = self.repository_wait();
7416        if !verify_current_head(&mut self.state, &winner, &wait).await? {
7417            return Ok(());
7418        }
7419        if self.state.gate_ran {
7420            // `review_loop` derives its conclusion from the clean review
7421            // record on every reentry and therefore puts a completed run back
7422            // in `Gating`. A recorded gate is a stronger, terminal fact:
7423            // retain its original command output (or lack of any, for a repo
7424            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
7425            // doc) and restore `Blocked` on a real failure rather than
7426            // pretending the command is still running or running it a second
7427            // time. `gate_ran == false` remains the only shape — unattempted,
7428            // or a resource-blocked retry — that may still need to execute a
7429            // command.
7430            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7431                self.state.status = RunStatus::Blocked;
7432                self.state.save()?;
7433            }
7434            return Ok(());
7435        }
7436        self.state.status = RunStatus::Gating;
7437        let mut outcomes = self.run_gate(&winner).await?;
7438        let mut attempted_head = self.state.gate_head.clone();
7439        loop {
7440            // A resource-blocked outcome means the gate command never actually
7441            // ran - the shared build cache could not be acquired or confirmed
7442            // fresh in time - which is evidence about the machine, not about
7443            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
7444            // Recording it as a red gate would mark a run `Blocked` on nothing
7445            // but contention magi has already logged; leaving `self.state.gate`
7446            // empty and `self.state.gate_ran` false instead keeps the shape
7447            // this function already treats as "still needs to run" (see the
7448            // early-return above), so the next call retries the command
7449            // rather than concluding anything.
7450            if verify_inconclusive(&outcomes) {
7451                self.state.save()?;
7452                return Ok(());
7453            }
7454            if outcomes.iter().all(CommandOutcome::ok) {
7455                break;
7456            }
7457            let fixes_before = self.state.gate_fixes.len();
7458            match self.gate_fix_round(&winner, &outcomes).await? {
7459                GateFix::Retry => {
7460                    outcomes = self.run_gate(&winner).await?;
7461                    attempted_head = self.state.gate_head.clone();
7462                }
7463                GateFix::Stop => {
7464                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
7465                    if self.state.gate_fixes.len() > fixes_before
7466                        && attempted_head.as_deref() != Some(current.as_str())
7467                        && verify_current_head(&mut self.state, &winner, &wait).await?
7468                    {
7469                        // A new SHA owes a gate even when a spent fixer changed only
7470                        // commit metadata. This does not grant another repair round.
7471                        outcomes = self.run_gate(&winner).await?;
7472                        attempted_head = self.state.gate_head.clone();
7473                    } else {
7474                        break;
7475                    }
7476                }
7477                GateFix::Defer => {
7478                    self.state.save()?;
7479                    return Ok(());
7480                }
7481            }
7482        }
7483        let passed = outcomes.iter().all(CommandOutcome::ok);
7484        self.state.gate = outcomes;
7485        self.state.gate_ran = true;
7486        self.state.gate_head = attempted_head;
7487        self.state.save()?;
7488        if !verify_current_head(&mut self.state, &winner, &wait).await? {
7489            return Ok(());
7490        }
7491        if !passed {
7492            self.state.status = RunStatus::Blocked;
7493            let spent = self.state.gate_fixes.len();
7494            self.state.event(
7495                "gate",
7496                if spent == 0 {
7497                    "gate failed; not merging".to_owned()
7498                } else {
7499                    format!("gate failed after {spent} gate-fix round(s); not merging")
7500                },
7501            );
7502        }
7503        self.state.save()?;
7504        Ok(())
7505    }
7506
7507    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7508    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7509    /// after review is clean and never on a candidate awaiting judging.
7510    ///
7511    /// Never fails the run: a non-zero exit or timeout is a warning and a
7512    /// recorded outcome, and the gate remains the single arbiter. Nothing
7513    /// configured means nothing happens - no event, no commit. `commit_all`
7514    /// commits any leftover change under the neutral identity and returns
7515    /// `false` when the tree is clean, so no empty commit is ever made.
7516    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7517        let commands = state.config.verify.pre_gate.clone();
7518        if commands.is_empty() {
7519            return;
7520        }
7521        let shell = state.config.shell();
7522        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7523        let (outcomes, _) = run_commands(
7524            state,
7525            "pre_gate",
7526            "pre_gate",
7527            0,
7528            &shell,
7529            &commands,
7530            &winner.worktree,
7531            timeout,
7532        )
7533        .await;
7534        for o in &outcomes {
7535            if !o.ok() {
7536                tracing::warn!(
7537                    "pre_gate `{}` failed ({:?}); the gate decides",
7538                    o.command,
7539                    o.code
7540                );
7541            }
7542            state.event(
7543                "pre_gate",
7544                format!(
7545                    "`{}` -> {}",
7546                    o.command,
7547                    if o.ok() {
7548                        "pass".to_owned()
7549                    } else {
7550                        format!(
7551                            "FAIL ({:?})\n{}",
7552                            o.code,
7553                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7554                        )
7555                    }
7556                ),
7557            );
7558        }
7559        state.pre_gate = outcomes;
7560        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7561            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7562                Ok(head) => {
7563                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7564                    state.pre_gate_commit = Some(head);
7565                }
7566                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7567            },
7568            Ok(false) => {}
7569            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7570        }
7571        if let Err(e) = state.save() {
7572            tracing::warn!("could not persist the pre_gate record: {e:#}");
7573        }
7574    }
7575
7576    /// Run `verify.gate` once against the winner's current tree, logging one
7577    /// event per command. Empty when nothing is configured.
7578    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7579        let wait = self.repository_wait();
7580        run_landing_gate(&mut self.state, winner, &wait).await
7581    }
7582
7583    /// One bounded fix round for a failing gate.
7584    ///
7585    /// The fixer is told the failure came from the gate itself, not from a
7586    /// reviewer, and is shown the failed commands, their exit codes and a tail
7587    /// of their output - whatever `[verify].gate` holds, nothing here knows
7588    /// what those commands run. Only a normal non-zero exit that printed
7589    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7590    /// command or a full disk says nothing about the code, and a fixer sent
7591    /// after it can only appease the machine. The round is judged by what git
7592    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7593    /// again before the gate does, so a fix cannot trade a green gate for a
7594    /// red e2e unnoticed.
7595    async fn gate_fix_round(
7596        &mut self,
7597        winner: &Candidate,
7598        outcomes: &[CommandOutcome],
7599    ) -> Result<GateFix> {
7600        let cap = self.state.config.graph.gate_fix_rounds;
7601        let spent = self.state.gate_fixes.len();
7602        if spent >= cap {
7603            if cap > 0 {
7604                self.state.event(
7605                    "gate",
7606                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7607                );
7608            }
7609            return Ok(GateFix::Stop);
7610        }
7611        if !gate_fixable(outcomes) {
7612            self.state.event(
7613                "gate",
7614                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7615                 command or similar); not spending a fix round on it",
7616            );
7617            return Ok(GateFix::Stop);
7618        }
7619        let min_free = self.state.config.disk.min_free_bytes;
7620        if min_free > 0 {
7621            match crate::disk::free_bytes(&winner.worktree) {
7622                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7623                Ok(free) => {
7624                    self.state.event(
7625                        "gate",
7626                        format!(
7627                            "only {free} bytes free ({min_free} required by `[disk] \
7628                             min_free_bytes`); not spending a fix round on a failure the disk \
7629                             may explain"
7630                        ),
7631                    );
7632                    return Ok(GateFix::Stop);
7633                }
7634                Err(e) => {
7635                    self.state.event(
7636                        "gate",
7637                        format!("free disk space could not be measured ({e:#}); no fix round"),
7638                    );
7639                    return Ok(GateFix::Stop);
7640                }
7641            }
7642        }
7643
7644        let attempt = spent + 1;
7645        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7646        let base = self.landing_base();
7647        let fix_prompt = prompt::gate_fix_in(
7648            &self.state.instruction,
7649            &failed,
7650            attempt,
7651            cap,
7652            &self.state.config.graph.language,
7653            &self.state.config.graph.github_language,
7654        );
7655        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7656        let sessions = self.state.config.graph.sessions;
7657        let artifacts = self.merge_repair_artifacts();
7658        self.state.event(
7659            "gate",
7660            format!("gate failed; gate-fix round {attempt} of {cap}"),
7661        );
7662        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7663        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7664        // Persist before invoking the fixer: it may commit and then be
7665        // interrupted before its result (or the following e2e) is saved.
7666        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7667            head: before.clone(),
7668            passed: false,
7669            outcomes: Vec::new(),
7670        });
7671        self.state.save()?;
7672        let (_, seat, out) = self
7673            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7674                prompt: fix_prompt.clone(),
7675                spec,
7676                seat,
7677                cwd: winner.worktree.clone(),
7678                timeout,
7679                allow_write: true,
7680                sessions,
7681                artifacts: artifacts.clone(),
7682                stem: format!("gate-fix-{attempt}"),
7683                handover: None,
7684            })
7685            .await;
7686        // The fixer was never asked: no round is spent (nothing is recorded)
7687        // and a later reentry asks the same agent again.
7688        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7689            return Ok(GateFix::Defer);
7690        }
7691        let mut record = GateFixRecord {
7692            agent: seat.agent.clone(),
7693            failed,
7694            notes: String::new(),
7695            committed: false,
7696            error: None,
7697        };
7698        match out {
7699            AgentOutcome::Ok(o) => {
7700                // A missing report is not a failed fix: the round is judged
7701                // by the tree below, and the report only carries prose.
7702                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7703                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7704                }
7705            }
7706            AgentOutcome::Dropped(_) => {
7707                record.error = Some("the CLI dropped the stream".to_owned());
7708            }
7709            AgentOutcome::Quota(o) => {
7710                self.state.quota.push(QuotaLoss {
7711                    seat: seat.key.clone(),
7712                    node: "gate-fix".to_owned(),
7713                    at: Timestamp::now(),
7714                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7715                });
7716                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7717            }
7718            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7719        }
7720        self.state.seats.insert(seat.key.clone(), seat);
7721        let coordination = self.coordinate(&winner.worktree).await?;
7722        if let Ok(r) = git::rescue_commit(
7723            &winner.worktree,
7724            &format!("magi: gate fix {attempt} (uncommitted work)"),
7725        )
7726        .await
7727        {
7728            self.state.note_withheld("gate-fix", &r.withheld);
7729        }
7730        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7731        record.committed = after != before;
7732        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7733        let note = record.error.clone();
7734        self.state.gate_fixes.push(record);
7735        self.state.save()?;
7736        if !changed {
7737            self.state.event(
7738                "gate",
7739                match note {
7740                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7741                    None => format!("gate-fix round {attempt}: the tree did not change"),
7742                },
7743            );
7744            return Ok(GateFix::Stop);
7745        }
7746        self.state.event(
7747            "gate",
7748            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7749        );
7750
7751        drop(coordination);
7752        let wait = self.repository_wait();
7753        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7754    }
7755
7756    // ---------------------------------------------------------------- merge
7757
7758    /// One read-only rewrite by the summary's author; a text the gate still
7759    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7760    /// applies on their word or on silence. `None` means the run is parked on
7761    /// that question and nothing may be pushed or opened yet.
7762    async fn guarded_pr_message(
7763        &mut self,
7764        winner: &Candidate,
7765        facts: Option<&BranchFacts>,
7766        posting: bool,
7767    ) -> Result<Option<PrMessage>> {
7768        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7769        let head = match final_head.as_deref() {
7770            Some(h) => crate::pr_outcome::FinalHead::At(h),
7771            None => crate::pr_outcome::FinalHead::Unknown,
7772        };
7773        // Only a pull request that is about to be posted pays for a summary.
7774        let summarized = if posting {
7775            Some(self.summarize_change(winner, final_head.as_deref()).await)
7776        } else {
7777            None
7778        };
7779        // A busy shared resource is not a permanent cause: publishing an
7780        // "unavailable" description would stick (a later head move never
7781        // edits the body). Stop before anything is pushed or opened; the
7782        // run is `Blocked` and a resume asks the summary again.
7783        if self.state.resource_block.is_some() {
7784            self.state.event(
7785                "pr-summary",
7786                "a shared resource was busy; nothing was published, resume to summarize again",
7787            );
7788            return Ok(None);
7789        }
7790        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7791            anyhow::bail!(
7792                "the final commit moved during public description generation; retry before publishing"
7793            );
7794        }
7795        let note = match (&summarized, &self.state.change_summary) {
7796            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7797            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7798            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7799            (None, _) => ChangeNote::NotAsked,
7800        };
7801        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7802        if !posting {
7803            let identity = crate::scrub::Identity::current();
7804            return Ok(Some(PrMessage {
7805                title: crate::scrub::scrub(&pr.title, &identity),
7806                body: crate::scrub::scrub(&pr.body, &identity),
7807            }));
7808        }
7809        // This very text was already rejected and asked about: no second
7810        // judge call, rewrite or question, just carry the record on.
7811        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7812        if let Some(known) = self
7813            .state
7814            .github_text
7815            .clone()
7816            .filter(|g| g.fingerprint == raw_fp)
7817        {
7818            return self.settle_withheld(winner, &pr, known).await;
7819        }
7820        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7821        let mut violations =
7822            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7823        let identity = crate::scrub::Identity::current();
7824        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7825            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7826            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7827        {
7828            violations.push(crate::github_text::Violation::SensitiveData);
7829        }
7830        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7831        // only a language violation starts one; sensitive data alone goes
7832        // straight to the owner, with the redacted text as the fallback.
7833        let has_language = violations
7834            .iter()
7835            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7836        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7837            self.state.event(
7838                "github-text",
7839                if has_language {
7840                    format!("description rejected: {violations:?}; requesting one rewrite")
7841                } else {
7842                    format!("description rejected: {violations:?}; asking the owner")
7843                },
7844            );
7845            let mut rewritten = false;
7846            let mut rejected_rewrite = None;
7847            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7848            {
7849                let output = text;
7850                if let Some(title) = summary_title(&output) {
7851                    // Judge the redacted candidate that will actually be posted.
7852                    // Sensitive data alone must not discard a useful rewrite.
7853                    let raw_body = summary_without_title(&output);
7854                    let clean_title = crate::scrub::scrub(&title, &identity);
7855                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7856                    let redacted = clean_title != title || body != raw_body;
7857                    let title = clean_title;
7858                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7859                    if !complete {
7860                        self.state.event(
7861                            "github-text",
7862                            "description rewrite omitted required findings",
7863                        );
7864                    }
7865                    if !body.trim().is_empty() {
7866                        body.push_str("\n\n");
7867                        body.push_str(&public_pr_evidence(&self.state, head));
7868                    }
7869                    let marker = format!("magi:run/{}", self.state.id);
7870                    if !body.trim().is_empty() && !body.contains(&marker) {
7871                        body.push_str(&format!("\n\n{marker}"));
7872                    }
7873                    let rewrite = PrMessage {
7874                        title: title.clone(),
7875                        body: body.clone(),
7876                    };
7877                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7878                    let mut failed = crate::github_text::check_for(
7879                        &self.state.config.graph,
7880                        &title,
7881                        &body,
7882                        redecision,
7883                    );
7884                    if (!crate::github_text::shareable(&title)
7885                        || !crate::github_text::shareable(&body))
7886                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7887                    {
7888                        failed.push(crate::github_text::Violation::SensitiveData);
7889                    }
7890                    if complete
7891                        && !title.trim().is_empty()
7892                        && !body.trim().is_empty()
7893                        && !failed.is_empty()
7894                    {
7895                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7896                            title: title.clone(),
7897                            body: body.clone(),
7898                            failed: failed
7899                                .iter()
7900                                .map(|v| crate::github_text::category(*v).to_owned())
7901                                .collect(),
7902                        });
7903                    }
7904                    if complete
7905                        && !title.trim().is_empty()
7906                        && !raw_body.trim().is_empty()
7907                        && failed.is_empty()
7908                    {
7909                        decision = redecision;
7910                        pr = PrMessage { title, body };
7911                        rewritten = true;
7912                        self.state
7913                            .event("github-text", "description rewrite passed");
7914                        if redacted {
7915                            self.state.event(
7916                                "github-text",
7917                                "sensitive data removed from description rewrite",
7918                            );
7919                        }
7920                    }
7921                }
7922            }
7923            if !rewritten {
7924                self.state.event(
7925                    "github-text",
7926                    "description rewrite unavailable or rejected; asking the owner",
7927                );
7928                if let Some(w) =
7929                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7930                {
7931                    // A different text than the one asked about before: that
7932                    // question no longer describes anything.
7933                    self.retire_github_text_question("the withheld text changed");
7934                    let record = crate::run::GithubTextAsk {
7935                        rewrite: rejected_rewrite,
7936                        fingerprint: raw_fp,
7937                        title: w.title,
7938                        body: w.body,
7939                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7940                        question: None,
7941                        asks: 0,
7942                        resolved: false,
7943                        chosen_title: None,
7944                        choice: None,
7945                    };
7946                    return self.settle_withheld(winner, &pr, record).await;
7947                }
7948            }
7949        }
7950        let (title, body) =
7951            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7952        Ok(Some(PrMessage { title, body }))
7953    }
7954
7955    /// Describe the whole change from its final diff in either run mode.
7956    ///
7957    /// The summary is tied to the final commit and to the merge base of the
7958    /// resolved remote base; a record for any other pair is dropped and the
7959    /// agent asked again, so a description never outlives the diff it was
7960    /// written from. The seat is read-only and the prompt tells it to read the
7961    /// diff and the code around it. `[roles] synthesizer` selects the agents
7962    /// (unset: the default pick), each asked at most once, advancing on an
7963    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7964    /// carries the reason it is unavailable; nothing is invented then.
7965    async fn summarize_change(
7966        &mut self,
7967        winner: &Candidate,
7968        head: Option<&str>,
7969    ) -> std::result::Result<(), String> {
7970        let Some(head) = head else {
7971            return Err("the final commit could not be read".to_owned());
7972        };
7973        let remote = self.state.config.merge.remote.clone();
7974        let tracking = format!("{remote}/{}", self.state.base_branch);
7975        if !git::rev_exists(&winner.worktree, &tracking).await {
7976            return Err(format!("the remote base `{tracking}` could not be read"));
7977        }
7978        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7979            Ok(b) if !b.is_empty() => b,
7980            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7981        };
7982        let evidence = crate::pr_outcome::summary_evidence(
7983            &self.state,
7984            crate::pr_outcome::FinalHead::At(head),
7985        );
7986        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7987        if self
7988            .state
7989            .change_summary
7990            .as_ref()
7991            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7992        {
7993            return Ok(());
7994        }
7995        if self.state.change_summary.take().is_some() {
7996            self.state.event(
7997                "pr-summary",
7998                "the final diff or evidence changed; the earlier change summary was discarded",
7999            );
8000        }
8001        // Save the source snapshot even if no generation agent is available.
8002        if let Err(e) = write_artifact(
8003            &self.state,
8004            &format!("public-description-{fingerprint}.md"),
8005            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8006        ) {
8007            tracing::warn!("could not save public description evidence: {e:#}");
8008        }
8009        let chain = match agent::pick_chain(
8010            &self.state.config.agents,
8011            self.state.config.roles.synthesizer.as_ref(),
8012            &agent::installed,
8013            "synthesizer",
8014        ) {
8015            Ok(chain) => chain,
8016            Err(e) => {
8017                tracing::warn!("no agent for the change summary: {e:#}");
8018                self.state.event("pr-summary", "no summary agent available");
8019                return Err("no agent was available to summarize it".to_owned());
8020            }
8021        };
8022        let subjects = git::subjects(&winner.worktree, &base, head)
8023            .await
8024            .unwrap_or_default();
8025        let stat = git::diff_stat(&winner.worktree, &base, head)
8026            .await
8027            .unwrap_or_default();
8028        let stat: String = stat.chars().take(6_000).collect();
8029        let mut prompt = crate::prompt::change_summary_in(
8030            &base,
8031            head,
8032            &subjects,
8033            stat.trim(),
8034            &self.state.config.graph.github_language,
8035        );
8036        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8037        let sessions = self.state.config.graph.sessions;
8038        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8039        let prompts = self.state.config.prompts.clone();
8040        let cache = self.state.config.cache_dir();
8041        let run = self.state.id.clone();
8042        for (i, spec) in chain.into_iter().enumerate() {
8043            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8044            let github_spec = match agent::for_github_text(
8045                &spec,
8046                &self.state.config.graph.github_language,
8047                &winner.worktree,
8048            ) {
8049                Ok(spec) => spec,
8050                Err(e) => {
8051                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8052                    continue;
8053                }
8054            };
8055            let job = SeatJob {
8056                spec: github_spec,
8057                seat,
8058                cwd: winner.worktree.clone(),
8059                prompt: prompt.clone(),
8060                timeout,
8061                allow_write: false,
8062                sessions,
8063                artifacts: agent::artifacts_dir(&self.state.dir()),
8064                stem: if i == 0 {
8065                    "change-summary".to_owned()
8066                } else {
8067                    format!("change-summary-{}", spec.id)
8068                },
8069                handover: None,
8070            };
8071            let ctx = WaveCtx {
8072                run: &run,
8073                node: "pr-summary",
8074                prompts: &prompts,
8075                cache: cache.as_deref(),
8076                round: None,
8077                carry_seats: false,
8078                pauses: Some((&self.pause, &self.interrupt)),
8079            };
8080            let (seat, outcome) =
8081                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8082            self.state.seats.insert(seat.key.clone(), seat);
8083            let text = match outcome {
8084                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8085                AgentOutcome::ResourceBlocked(why) => {
8086                    // No agent was asked: the run stops resumably before
8087                    // anything is published (see `guarded_pr_message`).
8088                    if self.state.resource_block.is_none() {
8089                        self.state.resource_block = Some(crate::run::ResourceBlock {
8090                            cause: crate::run::ResourceBlockCause::Other,
8091                            node: "pr-summary".to_owned(),
8092                            kind: "shared resource".to_owned(),
8093                            detail: why,
8094                            at: Timestamp::now(),
8095                        });
8096                    }
8097                    return Err("a shared resource was busy".to_owned());
8098                }
8099                _ => continue,
8100            };
8101            let blind = &self.state.config.blind;
8102            let Some(title) = summary_title(&text) else {
8103                continue;
8104            };
8105            let body: String = summary_without_title(&text)
8106                .lines()
8107                .filter(|l| !l.contains("magi:run/"))
8108                .collect::<Vec<_>>()
8109                .join("\n");
8110            let body = blind::sanitize_prose(body.trim(), blind);
8111            let title = blind::sanitize_prose(&title, blind);
8112            if body.trim().is_empty() || title.trim().is_empty() {
8113                continue;
8114            }
8115            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8116                continue;
8117            }
8118            // The branch must still be the one the agent was shown.
8119            if git::rev_parse(&winner.worktree, "HEAD")
8120                .await
8121                .ok()
8122                .as_deref()
8123                != Some(head)
8124            {
8125                return Err("the branch moved while the summary was written".to_owned());
8126            }
8127            if git::merge_base(&winner.worktree, &tracking, head)
8128                .await
8129                .ok()
8130                .as_deref()
8131                != Some(base.as_str())
8132                || crate::pr_outcome::summary_evidence(
8133                    &self.state,
8134                    crate::pr_outcome::FinalHead::At(head),
8135                ) != evidence
8136            {
8137                return Err("the summary evidence changed while it was written".to_owned());
8138            }
8139            self.state.change_summary = Some(crate::run::ChangeSummary {
8140                head: head.to_owned(),
8141                base,
8142                evidence: fingerprint,
8143                title,
8144                body,
8145            });
8146            self.state.event(
8147                "pr-summary",
8148                format!("change summary written by {} for the final diff", spec.id),
8149            );
8150            if let Err(e) = self.state.save() {
8151                tracing::warn!("could not save the change summary: {e:#}");
8152            }
8153            return Ok(());
8154        }
8155        self.state
8156            .event("pr-summary", "no agent produced a change summary");
8157        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8158    }
8159
8160    /// One read-only rewrite request for a withheld description. The
8161    /// implementer's own conversation is tried first when it exists and can be
8162    /// resumed; then the `[roles] language_judge` chain (or the default pick)
8163    /// writes it in a separate conversation, skipping the implementer's agent.
8164    /// The prompt carries the whole text, so no agent needs any memory. Each
8165    /// agent is asked at most once and the chain advances on an error, quota,
8166    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
8167    /// that has run out of quota must not end the rewrite. Nothing is recorded
8168    /// as a handover.
8169    async fn rewrite_pr_text(
8170        &mut self,
8171        winner: &Candidate,
8172        pr: &PrMessage,
8173        violations: &[crate::github_text::Violation],
8174    ) -> Option<String> {
8175        let prompt = format!(
8176            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8177            pr.title, pr.body
8178        );
8179        let sessions = self.state.config.graph.sessions;
8180        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8181        // (spec, the resumed implementer's seat). A fallback seat is created
8182        // only when its turn comes, so an unused fallback leaves no record.
8183        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8184        let key = format!("impl-{}", winner.label);
8185        if let Some(spec) = self
8186            .state
8187            .config
8188            .agents
8189            .iter()
8190            .find(|a| a.id == winner.agent)
8191            .cloned()
8192            && let Some(existing) = self.state.seats.get(&key)
8193            && existing.agent == spec.id
8194            && agent::has_session(spec.kind, existing, sessions)
8195        {
8196            attempts.push((spec, Some(existing.clone())));
8197        }
8198        match agent::pick_chain(
8199            &self.state.config.agents,
8200            self.state.config.roles.language_judge.as_ref(),
8201            &agent::installed,
8202            "language_judge",
8203        ) {
8204            Ok(chain) => {
8205                for spec in chain {
8206                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8207                        continue;
8208                    }
8209                    attempts.push((spec, None));
8210                }
8211            }
8212            Err(e) if attempts.is_empty() => {
8213                tracing::warn!("no agent for the description rewrite: {e:#}");
8214                self.state
8215                    .event("github-text", "no rewrite agent available");
8216                return None;
8217            }
8218            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8219        }
8220        let prompts = self.state.config.prompts.clone();
8221        let cache = self.state.config.cache_dir();
8222        let run = self.state.id.clone();
8223        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8224            let resumed = resumed_seat.is_some();
8225            let seat = resumed_seat.unwrap_or_else(|| {
8226                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8227            });
8228            let github_spec = match agent::for_github_text(
8229                &spec,
8230                &self.state.config.graph.github_language,
8231                &winner.worktree,
8232            ) {
8233                Ok(spec) => spec,
8234                Err(e) => {
8235                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8236                    continue;
8237                }
8238            };
8239            let job = SeatJob {
8240                spec: github_spec,
8241                seat,
8242                cwd: winner.worktree.clone(),
8243                prompt: prompt.clone(),
8244                timeout: retry_budget(timeout_full, resumed),
8245                allow_write: false,
8246                sessions,
8247                artifacts: agent::artifacts_dir(&self.state.dir()),
8248                stem: if i == 0 {
8249                    "github-text-rewrite".to_owned()
8250                } else {
8251                    format!("github-text-rewrite-{}", spec.id)
8252                },
8253                handover: None,
8254            };
8255            let ctx = WaveCtx {
8256                run: &run,
8257                node: "github-text",
8258                prompts: &prompts,
8259                cache: cache.as_deref(),
8260                round: None,
8261                carry_seats: false,
8262                pauses: Some((&self.pause, &self.interrupt)),
8263            };
8264            let (seat, outcome) =
8265                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8266            self.state.seats.insert(seat.key.clone(), seat);
8267            match outcome {
8268                AgentOutcome::Ok(o)
8269                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8270                {
8271                    return Some(o.text);
8272                }
8273                AgentOutcome::ResourceBlocked(_) => return None,
8274                _ => {}
8275            }
8276        }
8277        None
8278    }
8279
8280    /// Abandon the standing posting-gate question, if any.
8281    fn retire_github_text_question(&mut self, why: &str) {
8282        let Some(id) = self
8283            .state
8284            .github_text
8285            .as_ref()
8286            .and_then(|g| g.question.clone())
8287        else {
8288            return;
8289        };
8290        let _ = ask::Questions::open().update(&id, |q| {
8291            q.abandon(why);
8292            Ok(())
8293        });
8294    }
8295
8296    /// Carry a withheld text to its end: file the question, park while it is
8297    /// open, and on an answer, silence or timeout pick the text to post. The
8298    /// decision is saved before it is used, so it applies once.
8299    async fn settle_withheld(
8300        &mut self,
8301        winner: &Candidate,
8302        pr: &PrMessage,
8303        mut g: crate::run::GithubTextAsk,
8304    ) -> Result<Option<PrMessage>> {
8305        use crate::github_text as gt;
8306        let store = ask::Questions::open();
8307        let timeout = self.state.config.graph.answer_timeout;
8308        if !g.resolved {
8309            let mut standing = g
8310                .question
8311                .as_ref()
8312                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8313            if let Some(q) = standing.as_ref()
8314                && q.status.open()
8315                && gt::expired(q, timeout)
8316            {
8317                let why = format!("no answer within {}s of asking", timeout.max(1));
8318                standing = store
8319                    .update(&q.id, |q| {
8320                        q.abandon(&why);
8321                        Ok(())
8322                    })
8323                    .ok()
8324                    .map(|(q, ())| q);
8325            }
8326            let mut ask_again: Option<Option<Vec<String>>> = None;
8327            match standing {
8328                // Recorded but never written (a stop between the two saves),
8329                // or removed: ask again without spending the retry.
8330                None if g.asks <= 1 => {
8331                    g.asks = 0;
8332                    ask_again = Some(None);
8333                }
8334                None => {
8335                    g.resolved = true;
8336                    g.chosen_title = None;
8337                    g.choice = Some(GithubTextChoice::Fallback);
8338                }
8339                Some(q) if q.status.open() => {
8340                    self.park_on_github_text(&g)?;
8341                    return Ok(None);
8342                }
8343                Some(q) => match gt::read_reply(&q) {
8344                    gt::Reply::Fallback => {
8345                        g.resolved = true;
8346                        g.chosen_title = None;
8347                        g.choice = Some(GithubTextChoice::Fallback);
8348                    }
8349                    gt::Reply::Original => {
8350                        g.resolved = true;
8351                        g.chosen_title = None;
8352                        g.choice = Some(GithubTextChoice::Original);
8353                    }
8354                    gt::Reply::Rewrite => {
8355                        g.resolved = true;
8356                        g.chosen_title = None;
8357                        g.choice = Some(if g.rewrite.is_some() {
8358                            GithubTextChoice::Rewrite
8359                        } else {
8360                            GithubTextChoice::Fallback
8361                        });
8362                    }
8363                    gt::Reply::Title(text) => {
8364                        let first = text
8365                            .lines()
8366                            .map(str::trim)
8367                            .find(|l| !l.is_empty())
8368                            .unwrap_or("");
8369                        let decision = if first.is_empty() {
8370                            None
8371                        } else {
8372                            let candidate = PrMessage {
8373                                title: first.to_owned(),
8374                                body: String::new(),
8375                            };
8376                            self.judge_pr_language(&winner.worktree, &candidate).await
8377                        };
8378                        match gt::vet_title(
8379                            &text,
8380                            decision,
8381                            self.state.config.graph.github_english(),
8382                        ) {
8383                            Ok(title) => {
8384                                g.resolved = true;
8385                                g.chosen_title = Some(title);
8386                                g.choice = Some(GithubTextChoice::MyTitle);
8387                            }
8388                            Err(cats) if g.asks < 2 => {
8389                                ask_again = Some(Some(
8390                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8391                                ))
8392                            }
8393                            Err(_) => {
8394                                g.resolved = true;
8395                                g.chosen_title = None;
8396                                g.choice = Some(GithubTextChoice::Fallback);
8397                            }
8398                        }
8399                    }
8400                },
8401            }
8402            if let Some(rejected) = ask_again {
8403                // A retry (after a rejected replacement title) is still about
8404                // the original text: the same ORIGINAL / FALLBACK comparison.
8405                let w = gt::Withheld {
8406                    title: g.title,
8407                    body: g.body,
8408                    categories: g.categories.clone(),
8409                };
8410                // Named by the original text's fingerprint alone, so a resume
8411                // rewrites the same file and the detail stays identical.
8412                let artifact = match crate::run::write_artifact(
8413                    &self.state,
8414                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
8415                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8416                ) {
8417                    Ok(p) => Some(p),
8418                    Err(e) => {
8419                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
8420                        None
8421                    }
8422                };
8423                let (detail, comparison) = gt::question_content(
8424                    &w,
8425                    &pr.title,
8426                    &pr.body,
8427                    &self.state.id,
8428                    rejected.as_deref(),
8429                    artifact.as_deref(),
8430                    g.rewrite.as_ref(),
8431                );
8432                let mut q = ask::Question::new(
8433                    self.state.id.clone(),
8434                    gt::ASK_NODE.to_owned(),
8435                    gt::ASK_SEAT.to_owned(),
8436                    gt::question_summary(&self.state.config.graph.language, &w),
8437                    detail,
8438                    gt::question_choices(&w, g.rewrite.as_ref()),
8439                );
8440                q.posting_comparison = Some(comparison);
8441                q.answer_timeout = timeout;
8442                // Recorded before it is written: a stop in between leaves a
8443                // record naming a question the resume then files again.
8444                g.question = Some(q.id.clone());
8445                g.asks += 1;
8446                self.state.github_text = Some(g.clone());
8447                self.state.event(
8448                    "github-text",
8449                    format!("asking the owner about the withheld text ({})", q.short()),
8450                );
8451                self.state.save()?;
8452                store
8453                    .put_executable(&mut q)
8454                    .context("file the posting-gate question")?;
8455                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8456                    tracing::warn!(
8457                        "could not notify about posting-gate question {}: {e:#}",
8458                        q.short()
8459                    );
8460                }
8461                self.park_on_github_text(&g)?;
8462                return Ok(None);
8463            }
8464        }
8465        let resolved = g.chosen_title.clone();
8466        let original = g.choice == Some(GithubTextChoice::Original);
8467        // Field names only: the event log never records the text.
8468        self.state.event(
8469            "github-text",
8470            if original {
8471                format!(
8472                    "posting the original text for: {}",
8473                    match (g.title, g.body) {
8474                        (true, true) => "title, description",
8475                        (true, false) => "title",
8476                        _ => "description",
8477                    }
8478                )
8479            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8480                "posting the saved rewrite title and description".to_owned()
8481            } else if resolved.is_some() {
8482                "posting the owner's replacement title".to_owned()
8483            } else {
8484                "posting the neutral text for the withheld fields".to_owned()
8485            },
8486        );
8487        self.state.github_text = Some(g.clone());
8488        self.state.save()?;
8489        if g.choice == Some(GithubTextChoice::Rewrite)
8490            && let Some(r) = g.rewrite.as_ref()
8491        {
8492            return Ok(Some(PrMessage {
8493                title: r.title.clone(),
8494                body: r.body.clone(),
8495            }));
8496        }
8497        let w = gt::Withheld {
8498            title: g.title,
8499            body: g.body,
8500            categories: g.categories.clone(),
8501        };
8502        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8503        if let Some(t) = resolved {
8504            title = t;
8505        }
8506        // Every withheld field is now neutral or vetted English.
8507        let english = gt::LanguageDecision {
8508            title_english: true,
8509            body_english: true,
8510            body_complete: true,
8511        };
8512        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8513        if original {
8514            // The owner's explicit choice overrides the gate for the fields it
8515            // was made for; the others keep what the gate produced.
8516            if g.title {
8517                title = pr.title.clone();
8518            }
8519            if g.body {
8520                body = pr.body.clone();
8521            }
8522        }
8523        Ok(Some(PrMessage { title, body }))
8524    }
8525
8526    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8527        self.state.github_text = Some(g.clone());
8528        self.state.end_pass_parked(ParkCause::Question);
8529        self.state.parked = true;
8530        self.state.event(
8531            "github-text",
8532            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8533        );
8534        self.state.save()
8535    }
8536
8537    /// Ask `[roles] language_judge` about this text and record which source
8538    /// decided; the text itself is never recorded.
8539    async fn judge_pr_language(
8540        &mut self,
8541        cwd: &Path,
8542        pr: &PrMessage,
8543    ) -> Option<crate::github_text::LanguageDecision> {
8544        self.state.config.roles.language_judge.as_ref()?;
8545        if !self.state.config.graph.language_gate() {
8546            return None;
8547        }
8548        let decision =
8549            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8550        self.state.event(
8551            "github-text",
8552            if decision.is_some() {
8553                "language decided by the language judge"
8554            } else {
8555                "language judge unavailable; using built-in heuristics"
8556            },
8557        );
8558        decision
8559    }
8560
8561    async fn merge(&mut self) -> Result<()> {
8562        if self.resource_halt()? || self.hold_base_sync().await? {
8563            return Ok(());
8564        }
8565        // Same reasoning as `gate`: ask the review and gate records directly
8566        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8567        // skip can rewrite on reentry to something that no longer says
8568        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8569        // so a hand-off (open findings, green verification) reaches merge
8570        // exactly like a genuinely clean round does.
8571        //
8572        // A run resumed mid-`land` never reaches here at all: `execute`
8573        // recognises `RunStatus::Landing` before it even calls `prep`, and
8574        // routes straight to `run_land` instead. That has to happen a level
8575        // up from this function, not with a check in here, because
8576        // `review_loop`'s own status recomputation (see its doc) runs
8577        // *before* `merge` on every reentry and would otherwise overwrite
8578        // the `Landing` marker with `Gating` before this node ever saw it.
8579        if self
8580            .state
8581            .base_sync
8582            .as_ref()
8583            .is_some_and(|s| s.conflict.is_some())
8584            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8585                != Some(RunStatus::Gating)
8586            // `gate_ran == false` is not "passed" - `gate` leaves it false
8587            // both before it has ever run and when its last attempt was
8588            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8589            // permission to merge on nothing but the review record. Only a
8590            // gate that actually ran may proceed; an intentional local
8591            // review can also finish with verification explicitly not configured.
8592            // `RunState::gate_status` computes that distinction.
8593            || !self.state.gate_allows_progress()
8594        {
8595            return Ok(());
8596        }
8597        // This node's own record, not `status`: `status == Ready` is not
8598        // unique to the harmless `MergeMode::None` path this line was
8599        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8600        // run's PR was closed without merging — and on that run `mode` is
8601        // still `Pr`, so a reentry that fell through here would push and
8602        // open a second pull request. `self.state.merge` is set exactly once
8603        // this node (or `land`) has already produced a verdict, under every
8604        // mode, which is what "already done" actually means here.
8605        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8606            return Ok(());
8607        }
8608        let Some(winner) = self.state.winner().cloned() else {
8609            return Ok(());
8610        };
8611        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8612        if self.state.gate_head.as_deref() != Some(head.as_str()) {
8613            self.state.status = RunStatus::Blocked;
8614            self.state.event(
8615                "gate",
8616                "winner changed after gate; fresh verification required",
8617            );
8618            self.state.save()?;
8619            return Ok(());
8620        }
8621        let wait = self.repository_wait();
8622        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8623            return Ok(());
8624        }
8625        let repo = self.state.repo.clone();
8626        let base = self.state.base_branch.clone();
8627        let mode = self.state.config.merge.mode;
8628        let style = self.state.config.merge.style;
8629        let facts = if is_review_run(&self.state) {
8630            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8631            let start = review_base(
8632                &repo,
8633                &self.state.config.merge.remote,
8634                &base,
8635                &self.state.base_commit,
8636                &winner.branch,
8637            )
8638            .await;
8639            branch_facts(&repo, &start, &winner.branch).await
8640        } else {
8641            None
8642        };
8643        // `None` when the base could not be freshly read: then an adopted
8644        // pull request's title is left alone.
8645        let leaked = if is_review_run(&self.state) {
8646            leaked_subjects(&self.state, &winner.branch).await
8647        } else {
8648            Some(Vec::new())
8649        };
8650        // `None`: parked on the owner's word about a withheld title or body.
8651        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8652        // the resume arrives here again.
8653        let Some(pr) = self
8654            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8655            .await?
8656        else {
8657            return Ok(());
8658        };
8659        let message = pr.commit_message();
8660
8661        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
8662        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
8663            return Ok(());
8664        }
8665        let outcome = match mode {
8666            MergeMode::None => MergeOutcome {
8667                mode,
8668                ok: true,
8669                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8670                empty: false,
8671            },
8672            MergeMode::Pr | MergeMode::Local
8673                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8674            {
8675                MergeOutcome {
8676                    mode,
8677                    ok: false,
8678                    detail: empty_candidate_detail(&self.state, &base),
8679                    empty: true,
8680                }
8681            }
8682            MergeMode::Local => {
8683                let on = git::current_branch(&repo).await?;
8684                if on.as_deref() != Some(base.as_str()) {
8685                    MergeOutcome {
8686                        mode,
8687                        ok: false,
8688                        detail: format!(
8689                            "{} has {} checked out, not the base branch {base}",
8690                            repo.display(),
8691                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8692                        ),
8693                        empty: false,
8694                    }
8695                } else if !git::is_clean(&repo).await? {
8696                    MergeOutcome {
8697                        mode,
8698                        ok: false,
8699                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8700                        empty: false,
8701                    }
8702                } else {
8703                    let out = match style {
8704                        MergeStyle::Merge => {
8705                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8706                        }
8707                        MergeStyle::Squash => {
8708                            git::merge_squash(&repo, &winner.branch, &message).await?
8709                        }
8710                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8711                    };
8712                    MergeOutcome {
8713                        mode,
8714                        ok: out.ok(),
8715                        detail: if out.ok() { out.stdout } else { out.stderr },
8716                        empty: false,
8717                    }
8718                }
8719            }
8720            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8721                self.state.pr_publication_pending = true;
8722                self.state.save()?;
8723                let out =
8724                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8725                self.state.pr_publication_pending = out.is_err();
8726                MergeOutcome {
8727                    mode,
8728                    ok: out.is_ok(),
8729                    detail: match out {
8730                        Ok(url) => url,
8731                        Err(e) => format!("{e:#}"),
8732                    },
8733                    empty: false,
8734                }
8735            }
8736            MergeMode::Pr => {
8737                let remote = self.state.config.merge.remote.clone();
8738                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8739                let pushed = if self.state.foreign_checkout.is_some() {
8740                    let refspec = format!("{delivery_head}:refs/heads/{public}");
8741                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
8742                } else {
8743                    git::push(&winner.worktree, &remote, &winner.branch).await?
8744                };
8745                if !pushed.ok() {
8746                    MergeOutcome {
8747                        mode,
8748                        ok: false,
8749                        detail: pushed.stderr,
8750                        empty: false,
8751                    }
8752                } else {
8753                    if self.state.foreign_checkout.is_some() {
8754                        self.state.pr_pushed_head = Some(delivery_head.clone());
8755                    }
8756                    // A retry or resume of a run whose branch already has an
8757                    // open pull request adopts it rather than failing on a
8758                    // duplicate. Only this winner branch into this base:
8759                    // `branch_for` derives the name from the run id, so a
8760                    // different run's pull request never matches.
8761                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
8762                    let out = match pr_merge_plan(found) {
8763                        PrPlan::Create => {
8764                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
8765                                .await
8766                        }
8767                        PrPlan::Adopt { url, title } => {
8768                            self.state
8769                                .event("merge", format!("Pr: adopted open pull request {url}"));
8770                            if self.state.github_text.as_ref().is_some_and(|g| {
8771                                g.resolved
8772                                    && g.choice == Some(GithubTextChoice::Rewrite)
8773                                    && g.rewrite.is_some()
8774                            }) {
8775                                // The owner chose the entire pair, even if the
8776                                // existing title already matches or was edited.
8777                                land::set_pr_text_verbatim(
8778                                    &winner.worktree,
8779                                    &url,
8780                                    &pr.title,
8781                                    &pr.body,
8782                                )
8783                                .await
8784                                .map(|()| url)
8785                            } else {
8786                                if title != pr.title
8787                                    && (!is_review_run(&self.state)
8788                                        || leaked
8789                                            .as_deref()
8790                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8791                                    && let Err(e) =
8792                                        if self.state.github_text.as_ref().is_some_and(|g| {
8793                                            g.resolved
8794                                                && g.title
8795                                                && g.choice == Some(GithubTextChoice::Original)
8796                                        }) {
8797                                            // The owner chose to post this title unmodified.
8798                                            land::set_pr_title_verbatim(
8799                                                &winner.worktree,
8800                                                &url,
8801                                                &pr.title,
8802                                            )
8803                                            .await
8804                                        } else {
8805                                            land::set_pr_title(
8806                                                &mut self.state,
8807                                                &winner.worktree,
8808                                                &url,
8809                                                &pr.title,
8810                                            )
8811                                            .await
8812                                        }
8813                                {
8814                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8815                                    self.state
8816                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8817                                }
8818                                Ok(url)
8819                            }
8820                        }
8821                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8822                    };
8823                    match out {
8824                        Ok(url) => MergeOutcome {
8825                            mode,
8826                            ok: true,
8827                            detail: url,
8828                            empty: false,
8829                        },
8830                        Err(e) => MergeOutcome {
8831                            mode,
8832                            ok: false,
8833                            detail: e.to_string(),
8834                            empty: false,
8835                        },
8836                    }
8837                }
8838            }
8839        };
8840
8841        self.state.status = match (mode, outcome.ok) {
8842            (MergeMode::None, _) => RunStatus::Ready,
8843            (_, true) if outcome.empty => RunStatus::Ready,
8844            (MergeMode::Pr, true) => RunStatus::Ready,
8845            (_, true) => RunStatus::Merged,
8846            (_, false) => RunStatus::Blocked,
8847        };
8848        self.state.event(
8849            "merge",
8850            format!(
8851                "{:?}: {}",
8852                mode,
8853                outcome.detail.lines().next().unwrap_or("")
8854            ),
8855        );
8856        self.state.merge = Some(outcome);
8857        self.state.save()?;
8858
8859        // The PR is open and the run would historically stop here, leaving the
8860        // operator to watch checks, feed review comments back to a fixer, and
8861        // merge. That was done by hand six times in one session before this
8862        // existed. Opt-in, because merging is the one irreversible thing magi
8863        // can do to a repository.
8864        if self.state.config.graph.land
8865            && mode == MergeMode::Pr
8866            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8867        {
8868            self.run_land().await?;
8869        }
8870        // `run_land` may have left `status` at `Landing` - still waiting on
8871        // CI or the owner's approval, not actually settled - so this has to
8872        // read whatever `status` ended up as here, not the `Merged` this
8873        // function set a few lines up.
8874        self.settle_questions();
8875        Ok(())
8876    }
8877
8878    /// Enter `land`.
8879    ///
8880    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8881    /// a resumed run's re-entry. `land::land` itself is what serialises the
8882    /// two git-mutating moments inside the loop — the rebase push and
8883    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8884    /// to hold a lock across the whole call, and doing so would serialise
8885    /// this run's CI wait against a *different* run's land-approval resume
8886    /// in the same repository, which is exactly the "must not wait on
8887    /// another task" property the daemon's slot-freeing exists to give.
8888    async fn run_land(&mut self) -> Result<()> {
8889        crate::pr_head::ensure_owner(&self.state)?;
8890        let url = self
8891            .state
8892            .pr
8893            .as_ref()
8894            .map(|pr| pr.url.clone())
8895            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8896            .unwrap_or_default();
8897        let url = url.lines().next().unwrap_or("").trim().to_owned();
8898        if !url.starts_with("http") {
8899            return Ok(());
8900        }
8901        // A land failure is not a lost run: the work is on a branch and the
8902        // pull request is open, which is exactly where a human takes over.
8903        loop {
8904            match land::land(&mut self.state, &url).await {
8905                Ok(_)
8906                    if self
8907                        .state
8908                        .merge_repairs
8909                        .last()
8910                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8911                {
8912                    if self.repair_merge_review().await? {
8913                        continue;
8914                    }
8915                }
8916                Ok(pr) if self.state.parked => {
8917                    // `land` already saved the parked marker; nothing here
8918                    // overrides `status` back to a terminal value while an
8919                    // approval is still outstanding.
8920                    let _ = pr;
8921                }
8922                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8923                    self.state.status = RunStatus::Superseded;
8924                    self.state.save()?;
8925                }
8926                Ok(pr) => {
8927                    self.state.status = match pr.state {
8928                        land::PrLifecycle::Merged => RunStatus::Merged,
8929                        _ => RunStatus::Blocked,
8930                    };
8931                    // Downstream of a confirmed merge only - see
8932                    // `bump::should_release_bump`'s own doc for why this one
8933                    // check covers all three of `land`'s success paths.
8934                    // Best-effort: the run already landed, so a failure here
8935                    // (the decision call, `gh`, `cargo`) is recorded and never
8936                    // turns a landed run into a failed one.
8937                    if bump::should_release_bump(self.state.status)
8938                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8939                    {
8940                        // The event is the run's own record. Not-eligible cases
8941                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8942                        // `Err` is a bump that was tried and failed:
8943                        // `after_merge` itself raises the operator notice for
8944                        // that, whether or not a release PR exists yet.
8945                        self.state
8946                            .event("bump", format!("release bump skipped: {e:#}"));
8947                    }
8948                    // Independent of the bump, and best-effort in the same way:
8949                    // findings the merge left open become follow-up tasks.
8950                    if self.state.status == RunStatus::Merged {
8951                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8952                    }
8953                    self.state.save()?;
8954                }
8955                Err(e) => {
8956                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8957                        return Err(e);
8958                    }
8959                    self.state.status = RunStatus::Blocked;
8960                    self.state.event("land", format!("gave up: {e}"));
8961                    self.state.save()?;
8962                }
8963            }
8964            break;
8965        }
8966        Ok(())
8967    }
8968
8969    /// One repair cycle stays under the existing driver/task claim. An
8970    /// interrupted fixer is rolled back and held rather than run twice.
8971    async fn repair_merge_review(&mut self) -> Result<bool> {
8972        use crate::run::MergeRepairPhase as Phase;
8973        crate::pr_head::ensure_owner(&self.state)?;
8974        let i = self.state.merge_repairs.len() - 1;
8975        let request = self.state.merge_repairs[i].clone();
8976        if request.phase == Phase::Fixing {
8977            let restored = match self.state.winner().cloned() {
8978                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8979                None => Err(anyhow::anyhow!("repair has no winning branch")),
8980            };
8981            let why = match restored {
8982                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8983                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8984                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8985            };
8986            self.hold_merge_repair(i, why)?;
8987            return Ok(false);
8988        }
8989        // A completed repair gate is a publication checkpoint. On reentry,
8990        // reject external changes before the ordinary gate path can replace
8991        // that evidence by verifying a new head. Requested repairs still
8992        // start their own review generation below.
8993        if request.phase == Phase::Reviewing && self.state.gate_ran {
8994            let winner = self
8995                .state
8996                .winner()
8997                .context("repair has no winning branch")?;
8998            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8999            if self.state.gate_head.as_deref() != Some(head.as_str())
9000                || !git::is_clean(&winner.worktree).await?
9001            {
9002                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9003                return Ok(false);
9004            }
9005        }
9006        let failure = if i >= self.state.config.graph.land_rounds
9007            || self.state.config.graph.review_rounds == 0
9008        {
9009            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9010        } else {
9011            None
9012        };
9013        if let Some(why) = failure {
9014            self.hold_merge_repair(i, why)?;
9015            return Ok(false);
9016        }
9017        if request.phase == Phase::Requested {
9018            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9019                Ok(pr) => pr,
9020                Err(e) => {
9021                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9022                    return Ok(false);
9023                }
9024            };
9025            self.state.merge_repairs[i].phase = Phase::Fixing;
9026            self.state.land_approval = None;
9027            self.state.land_armed_head = None;
9028            self.state.save()?;
9029            // Serialize the exact records, including finding ids/details,
9030            // reconsideration reasons, final votes and verification output.
9031            let evidence = serde_json::to_string_pretty(&request)?;
9032            let reason = format!(
9033                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9034            );
9035            let budget = self.state.config.graph.land_rounds;
9036            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9037            match result {
9038                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9039                Ok(land::Fixed::Failed(why)) => {
9040                    self.hold_merge_repair(
9041                        i,
9042                        format!(
9043                            "repair failed: {why}; inspect the branch and run an explicit review"
9044                        ),
9045                    )?;
9046                    return Ok(false);
9047                }
9048                Err(e) => {
9049                    self.hold_merge_repair(
9050                        i,
9051                        format!(
9052                            "repair failed: {e:#}; inspect the branch and run an explicit review"
9053                        ),
9054                    )?;
9055                    return Ok(false);
9056                }
9057            }
9058            // Even no change gets a real new reviewer wave. The previous
9059            // evidence remains in the request; a former clean/concluded round
9060            // cannot short-circuit the new review or gate.
9061            self.begin_merge_repair_review(i)?;
9062        }
9063        self.review_loop().await?;
9064        self.gate().await?;
9065        if self.resource_halt()? || self.state.parked {
9066            return Ok(false);
9067        }
9068        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9069            != Some(RunStatus::Gating)
9070            || !self.state.gate_allows_progress()
9071        {
9072            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9073            return Ok(false);
9074        }
9075        // Review and gate may have added commits after the initial repair.
9076        // Publish exactly their final head to the existing PR, with a lease.
9077        let winner = self
9078            .state
9079            .winner()
9080            .cloned()
9081            .context("repair has no winning branch")?;
9082        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9083        if self.state.gate_head.as_deref() != Some(after.as_str())
9084            || !git::is_clean(&winner.worktree).await?
9085        {
9086            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9087            return Ok(false);
9088        }
9089        let expected = self
9090            .state
9091            .pr_pushed_head
9092            .as_deref()
9093            .unwrap_or(&request.head)
9094            .to_owned();
9095        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9096        // Push the checked commit itself: a local ref moving after the check
9097        // must not change which commit the forge receives.
9098        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9099            return Ok(false);
9100        }
9101        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9102        let refspec = format!("{after}:refs/heads/{public}");
9103        let push = git::git_raw(
9104            &winner.worktree,
9105            &["push", &lease, &self.state.config.merge.remote, &refspec],
9106        )
9107        .await;
9108        match push {
9109            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9110            Ok(out) => {
9111                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9112                return Ok(false);
9113            }
9114            Err(e) => {
9115                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9116                return Ok(false);
9117            }
9118        }
9119        self.state.merge_repairs[i].phase = Phase::Complete;
9120        self.state.status = RunStatus::Landing;
9121        self.state.event(
9122            "repair",
9123            "new review and gate completed; fresh owner merge decision required",
9124        );
9125        self.state.save()?;
9126        Ok(true)
9127    }
9128
9129    fn merge_repair_artifacts(&self) -> PathBuf {
9130        let artifacts = agent::artifacts_dir(&self.state.dir());
9131        match self
9132            .state
9133            .merge_repairs
9134            .last()
9135            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9136        {
9137            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9138            None => artifacts,
9139        }
9140    }
9141
9142    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9143        if let Some(e2e) = self.state.head_e2e.take() {
9144            self.state
9145                .verification_history
9146                .push(crate::run::VerificationEvidence {
9147                    superseded_by: None,
9148                    archived_at: Some(Timestamp::now()),
9149                    fixes_spent: Some(self.state.gate_fixes.len()),
9150                    gate_head: None,
9151                    gate_ran: false,
9152                    gate: Vec::new(),
9153                    e2e: Some(e2e),
9154                });
9155        }
9156        self.state.reviews.clear();
9157        self.state.gate.clear();
9158        self.state.gate_ran = false;
9159        self.state.gate_head = None;
9160        self.state.gate_fixes.clear();
9161        self.state.gate_repair_e2e = None;
9162        self.state.pre_gate.clear();
9163        self.state.pre_gate_commit = None;
9164        self.state.contested_handoff = None;
9165        self.state.land_reruns = None;
9166        self.state.land_approval = None;
9167        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9168        self.state.status = RunStatus::Reviewing;
9169        self.state.save()
9170    }
9171
9172    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9173        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9174        self.state.merge_repairs[index].failure = Some(why.clone());
9175        self.state.status = RunStatus::Blocked;
9176        self.state.event("repair", why);
9177        self.state.save()
9178    }
9179
9180    // -------------------------------------------------------------- helpers
9181
9182    /// Fetch or create a seat, keeping its conversation across nodes.
9183    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9184        if let Some(existing) = self.state.seats.get(key)
9185            && existing.agent == agent
9186        {
9187            return existing.clone();
9188        }
9189        // A seat that changes agent mints its session id from the agent too,
9190        // like a handover: the old agent's uuid is already taken by the CLI.
9191        let fresh = if self.state.seats.contains_key(key) {
9192            handover_seat(key, agent, self.state.next_seat_seed())
9193        } else {
9194            SeatState::new(key, agent, self.state.seed)
9195        };
9196        self.state.seats.insert(key.to_owned(), fresh.clone());
9197        fresh
9198    }
9199
9200    /// The agent now holding seat `key`: `spec`, unless a handover moved the
9201    /// seat to another roster agent, in which case that agent. Nodes that
9202    /// continue a seat's conversation (deliberation, the votes, a reviewer's
9203    /// reconsideration) must keep talking to whoever answered it, not slip
9204    /// back to the agent that failed it.
9205    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9206        match self.state.seats.get(key) {
9207            Some(s) if s.agent != spec.id => {
9208                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9209            }
9210            _ => spec,
9211        }
9212    }
9213
9214    /// A candidate rendered for judging, with the leak policy applied.
9215    fn view(&self, c: &Candidate) -> CandidateView {
9216        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9217            .unwrap_or_default();
9218        let (patch, _) = blind::sanitize_patch(
9219            &format!("candidate {} patch", c.label),
9220            &raw,
9221            &self.state.config.blind,
9222        );
9223        CandidateView {
9224            label: c.label,
9225            branch: c.branch.clone(),
9226            summary: c.summary.clone(),
9227            stat: c.stat.clone(),
9228            patch,
9229        }
9230    }
9231
9232    /// The full candidate set as prompt text, for seats with no live session.
9233    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9234        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9235        prompt::judge(
9236            "(see above)",
9237            &views,
9238            self.roles.judges.len(),
9239            base_short,
9240            "en",
9241        )
9242    }
9243
9244    /// The final-vote prompt with everything a seat that has no session of its
9245    /// own needs: the candidates, the seat's own ranking and reasons, and the
9246    /// anonymised deliberation it took part in (only when there was one, so a
9247    /// handed-over seat never sees more than the seat it replaces did). The
9248    /// `Final vote` heading stays first.
9249    fn vote_prompt_full(
9250        &self,
9251        j: usize,
9252        labels: &[char],
9253        language: &str,
9254        candidates: &[Candidate],
9255        base_short: &str,
9256    ) -> String {
9257        let mut text = format!(
9258            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9259            prompt::final_vote(labels, language),
9260            self.state.instruction,
9261            self.candidate_block(candidates, base_short)
9262        );
9263        if let Some(own) = self
9264            .state
9265            .judgements
9266            .get(j)
9267            .filter(|r| !r.ranking.is_empty())
9268        {
9269            let reasons = own
9270                .reasons
9271                .iter()
9272                .map(|(k, v)| format!("- {k}: {v}"))
9273                .collect::<Vec<_>>()
9274                .join("\n");
9275            text.push_str(&format!(
9276                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9277                own.ranking.iter().collect::<String>(),
9278                if reasons.is_empty() {
9279                    ""
9280                } else {
9281                    ", because:\n"
9282                }
9283            ));
9284        }
9285        if !self.state.deliberation.is_empty() {
9286            text.push_str("\n# What was argued before this vote\n");
9287            for t in self.transcript(&[], j) {
9288                text.push_str(&format!(
9289                    "\n## {}{}\n\n{}\n",
9290                    t.who,
9291                    if t.is_self { " (you)" } else { "" },
9292                    t.body.trim()
9293                ));
9294            }
9295        }
9296        text
9297    }
9298
9299    /// Anonymised transcript for judge `self_idx`.
9300    ///
9301    /// The initial rankings are always the opening statements. Seeding them
9302    /// only when no turn had been taken yet meant every judge after the first
9303    /// argued against a single voice instead of against the actual split — the
9304    /// disagreement is the information, so it is always on the table.
9305    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9306        let mut turns = Vec::new();
9307        for j in &self.state.judgements {
9308            if j.ranking.is_empty() {
9309                continue;
9310            }
9311            let reasons = j
9312                .reasons
9313                .iter()
9314                .map(|(k, v)| format!("- {k}: {v}"))
9315                .collect::<Vec<_>>()
9316                .join("\n");
9317            turns.push(Turn {
9318                who: format!("Judge {} (opening ranking)", j.judge),
9319                is_self: j.judge == self_idx + 1,
9320                body: format!(
9321                    "Ranked {}{}{reasons}",
9322                    j.ranking.iter().collect::<String>(),
9323                    if reasons.is_empty() {
9324                        ""
9325                    } else {
9326                        ", because:\n"
9327                    }
9328                ),
9329            });
9330        }
9331        for t in self
9332            .state
9333            .deliberation
9334            .iter()
9335            .flat_map(|r| r.turns.iter())
9336            .chain(current)
9337        {
9338            turns.push(Turn {
9339                who: format!("Judge {}", t.judge),
9340                is_self: t.judge == self_idx + 1,
9341                body: t.body.clone(),
9342            });
9343        }
9344        turns
9345    }
9346}
9347
9348/// Does this seat still hold the context a follow-up prompt would rely on?
9349fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9350    agent::has_session(spec.kind, seat, sessions)
9351}
9352
9353/// The next entry in `roster` after `start`, never wrapping back to the
9354/// front, whose id is not in `tried` yet.
9355///
9356/// Starts one past `start` rather than at the front of `roster`: `start` is
9357/// the seat's own original position, and a seat whose candidate slot already
9358/// sits on the roster's second entry must fall through to the third next, not
9359/// restart at the first — which is very likely a different candidate's own
9360/// agent already. Never wraps back past `start`, for the same reason: an
9361/// entry earlier in the roster than the seat's own position is almost
9362/// certainly some *other* candidate slot's own agent, and once the tail of
9363/// the roster is exhausted there are no more untried agents for *this* seat
9364/// to fall through to — the caller's fallback chain ends there, exactly as
9365/// "no further untried agents remain in the list for that seat" asks for.
9366///
9367/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
9368/// the same id twice (an operator's `roles.implementers` typo, or a
9369/// `[[agents]]` list reused across roles) must not let
9370/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
9371/// over `roster` either finds an untried id or runs out, so this always
9372/// terminates regardless of duplicates.
9373fn next_untried_in_roster<'a>(
9374    roster: &'a [AgentSpec],
9375    start: usize,
9376    tried: &BTreeSet<String>,
9377) -> Option<&'a AgentSpec> {
9378    roster
9379        .get(start + 1..)?
9380        .iter()
9381        .find(|s| !tried.contains(&s.id))
9382}
9383
9384/// The successor for a seat, shared by all four seat kinds (implement, judge,
9385/// review, advise). `others` holds the ids that *currently* occupy the other
9386/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
9387/// sees them), so roster entries beyond the seat count act as spares: a failed
9388/// seat goes to an agent no other seat holds whenever the roster permits.
9389///
9390/// `carried` is `Some` only for the review loop's carried failure history.
9391/// Order: (1) forward from `start`, never wrapping, untried, not a carried
9392/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
9393/// whole roster: untried and not another seat's occupant; (3) the plain walk
9394/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
9395/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
9396/// a better outcome than an empty seat, and it keeps the answer to "is there
9397/// a successor at all" exactly what it was before occupants were considered,
9398/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
9399/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
9400/// carried failure is a cheaper bet than doubling an agent on the panel, and
9401/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
9402/// the same round are handled one at a time, so a seat later in the batch
9403/// sees an earlier one's new occupant but not yet its own freed agent.
9404fn pick_successor<'a>(
9405    roster: &'a [AgentSpec],
9406    start: usize,
9407    tried: &BTreeSet<String>,
9408    carried: Option<&BTreeSet<String>>,
9409    others: &BTreeSet<String>,
9410) -> Option<&'a AgentSpec> {
9411    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9412    roster
9413        .get(start + 1..)
9414        .and_then(|tail| {
9415            tail.iter()
9416                .filter(free)
9417                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9418        })
9419        .or_else(|| {
9420            carried
9421                .is_some()
9422                .then(|| roster.iter().find(free))
9423                .flatten()
9424        })
9425        .or_else(|| match carried {
9426            Some(c) => next_for_seat(roster, start, tried, c),
9427            None => next_untried_in_roster(roster, start, tried),
9428        })
9429}
9430
9431/// The next agent for a seat that carries its failure history across rounds
9432/// (the review loop). `round_tried` is this round's own bound and starts
9433/// empty every round; `carried_failed` only decides priority.
9434///
9435/// First: an id walking forward from `start`, never wrapping, that is neither
9436/// tried this round nor failed in an earlier one. Only when that is exhausted
9437/// does it rescue: the first roster id (in roster order, so this one *does*
9438/// look before `start`) not yet tried this round, which is by then a carried
9439/// failure. Each id is rescued at most once per round, so it cannot loop.
9440fn next_for_seat<'a>(
9441    roster: &'a [AgentSpec],
9442    start: usize,
9443    round_tried: &BTreeSet<String>,
9444    carried_failed: &BTreeSet<String>,
9445) -> Option<&'a AgentSpec> {
9446    roster
9447        .get(start + 1..)?
9448        .iter()
9449        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9450        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9451}
9452
9453/// Where a reviewer seat starts a round: the agent that last answered it when
9454/// it is still on the roster and not marked failed, else the spec's own agent
9455/// unless it failed, else the next roster agent that has not failed, else the
9456/// spec's own agent again (the whole roster failed). Ids no longer on the
9457/// roster are ignored. An empty roster has no handover, so the spec stands.
9458fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9459    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9460        return spec;
9461    };
9462    let ok = |id: &str| !h.failed.contains(id);
9463    if let Some(last) = h.last_ok.as_deref()
9464        && ok(last)
9465        && let Some(s) = roster.iter().find(|s| s.id == last)
9466    {
9467        return s.clone();
9468    }
9469    if ok(&spec.id) {
9470        return spec;
9471    }
9472    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9473    roster
9474        .get(from + 1..)
9475        .into_iter()
9476        .flatten()
9477        .chain(roster.iter())
9478        .find(|s| ok(&s.id))
9479        .cloned()
9480        .unwrap_or(spec)
9481}
9482
9483/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
9484/// seed so a CLI that mints its session id up front (`--session-id`) never
9485/// reuses the uuid the previous agent already opened under the same seat key.
9486pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9487    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9488}
9489
9490/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
9491/// so the classifier below and the code that builds the message cannot drift.
9492const TIMED_OUT: &str = "timed out";
9493
9494impl FailClass {
9495    /// `None` for an answer; otherwise how the turn failed.
9496    fn of(out: &AgentOutcome) -> Option<Self> {
9497        match out {
9498            AgentOutcome::Ok(_) => None,
9499            AgentOutcome::Quota(_) => Some(Self::Quota),
9500            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9501            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9502            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9503            // Not a failure of the agent. Callers check for it first; `None`
9504            // here means "no class to hand over on".
9505            AgentOutcome::ResourceBlocked(_) => None,
9506        }
9507    }
9508
9509    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
9510    fn stem_word(&self) -> &'static str {
9511        match self {
9512            Self::Quota => "quota",
9513            _ => "handover",
9514        }
9515    }
9516}
9517
9518/// The message's first line with its variable parts removed — digit runs and
9519/// path-like tokens — so "exit code 2" and "exit code 7" read
9520/// as one kind of failure.
9521fn failure_signature(msg: &str) -> String {
9522    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9523    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
9524    // the historical machine key while the process diagnosis uses readable prose.
9525    if let Some(code) = line
9526        .strip_prefix("process exited with no usable output (exit code ")
9527        .and_then(|s| s.strip_suffix(')'))
9528        .and_then(|s| s.parse::<i32>().ok())
9529    {
9530        line = format!("exited with some({code}) and no usable output");
9531    } else if matches!(
9532        line.as_str(),
9533        "process exited with no usable output (no exit code reported)"
9534            | "process exited with no usable output (exit code unavailable)"
9535    ) {
9536        line = "exited with none and no usable output".to_owned();
9537    }
9538    let mut out = Vec::new();
9539    for word in line.split_whitespace() {
9540        if word.contains('/') || word.contains('\\') {
9541            out.push("<path>".to_owned());
9542            continue;
9543        }
9544        let mut w = String::new();
9545        let mut in_digits = false;
9546        for c in word.chars() {
9547            if c.is_ascii_digit() {
9548                if !in_digits {
9549                    w.push('#');
9550                }
9551                in_digits = true;
9552            } else {
9553                in_digits = false;
9554                w.push(c);
9555            }
9556        }
9557        out.push(w);
9558    }
9559    out.join(" ").chars().take(120).collect()
9560}
9561
9562/// Whether a seat that just failed with `cur` may go to the next roster agent.
9563/// A quota or a timeout always may. Any other failure may not when the agent
9564/// before it failed the same way: an error the prompt causes would otherwise
9565/// walk the whole roster. `prev` is the class of the immediately preceding
9566/// agent's failure, so a quota or timeout in between breaks the run of
9567/// identical failures by itself.
9568fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9569    match cur {
9570        FailClass::Quota | FailClass::Timeout => true,
9571        FailClass::Other(_) => prev != Some(cur),
9572    }
9573}
9574
9575/// A short human reason for a failed outcome, for the handover record.
9576fn fail_reason(out: &AgentOutcome) -> String {
9577    match out {
9578        AgentOutcome::Ok(_) => String::new(),
9579        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9580        AgentOutcome::Dropped(o) => format!(
9581            "the CLI dropped the stream ({})",
9582            o.dropped
9583                .as_ref()
9584                .map(|d| d.why.as_str())
9585                .unwrap_or("it ended without delivering its answer")
9586        ),
9587        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9588            e.lines().next().unwrap_or("").chars().take(160).collect()
9589        }
9590    }
9591}
9592
9593/// Note one handover in the run: the structured record and, in the timeline,
9594/// the sentence a person reads. A quota keeps the wording it always had.
9595pub(crate) fn record_handover(
9596    state: &mut RunState,
9597    node: &str,
9598    seat: &str,
9599    from: &str,
9600    to: &str,
9601    class: &FailClass,
9602    reason: &str,
9603) {
9604    let message = if *class == FailClass::Quota {
9605        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9606    } else {
9607        format!("{seat}: handed over {from} -> {to} ({reason})")
9608    };
9609    state.event(node, message);
9610    state.handovers.push(Handover {
9611        at: Timestamp::now(),
9612        node: node.to_owned(),
9613        seat: seat.to_owned(),
9614        from: from.to_owned(),
9615        to: to.to_owned(),
9616        reason: reason.to_owned(),
9617    });
9618}
9619
9620/// Did this reply report running a command whose own CLI never confirmed an
9621/// exit status?
9622///
9623/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9624/// command *finished* (see that type's own doc), so this can only be `true`
9625/// for a command whose completion event carried no readable exit code — not
9626/// for one that simply is not mentioned at all. That is the one signal this
9627/// crate can read, from the same record `state.jobs` renders, about a reply
9628/// standing next to work its own CLI cannot vouch for finishing; it is
9629/// deliberately not a check on the exit code's *value* (a fixer legitimately
9630/// runs a command that fails mid-iteration before it succeeds) and not a
9631/// guess at a command still running in the background (which emits no event
9632/// at all, and so leaves no evidence here to find).
9633fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9634    commands.iter().any(|c| c.exit_code.is_none())
9635}
9636
9637/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9638/// trusted as a verified no-op — the adoption guard's own text-level half.
9639///
9640/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9641/// timed out): a marker only earns the benefit of the doubt from a turn the
9642/// CLI itself vouches for finishing properly, the same house style
9643/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9644/// *fix* report to for `commands`. A candidate that timed out, exited
9645/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9646/// is, whatever prose it wrote — this returns `None` before it ever looks at
9647/// `text`. The remaining guards (the tree really is empty, the evidence is
9648/// non-empty) are the caller's: this only reads what the reply *claimed*.
9649fn verified_noop_claim(
9650    usable: bool,
9651    commands: &[agent::CommandEvidence],
9652    text: &str,
9653) -> Option<String> {
9654    (usable && !has_unconfirmed_command(commands))
9655        .then(|| verdict::verified_noop(text))
9656        .flatten()
9657}
9658
9659fn short(commit: &str) -> String {
9660    commit.chars().take(7).collect()
9661}
9662
9663fn make_executable(path: &Path) -> Result<()> {
9664    #[cfg(unix)]
9665    {
9666        use std::os::unix::fs::PermissionsExt as _;
9667        let mut perms = std::fs::metadata(path)?.permissions();
9668        perms.set_mode(0o755);
9669        std::fs::set_permissions(path, perms)?;
9670    }
9671    #[cfg(not(unix))]
9672    {
9673        let _ = path;
9674    }
9675    Ok(())
9676}
9677
9678/// What every seat in one batch shares: where the answers are attributed, the
9679/// prompt overlay they inherit, and the build cache they are told to use.
9680///
9681/// A struct rather than four more parameters: `wave` also needs the run's
9682/// state (to record who is answering right now) and the attempt number, and
9683/// eight positional arguments is both unreadable and a clippy error.
9684struct WaveCtx<'a> {
9685    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9686    /// paid for it.
9687    run: &'a str,
9688    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9689    node: &'a str,
9690    prompts: &'a Prompts,
9691    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9692    cache: Option<&'a Path>,
9693    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9694    /// `JobRecord::round`. `None` for every other node.
9695    round: Option<usize>,
9696    /// Carry each seat's failed-agent history across waves (the review loop
9697    /// only): start-of-round priority and handover choice read
9698    /// [`RunState::seat_history`], and every answer or failure writes it.
9699    carry_seats: bool,
9700    /// The run's park / interrupt requests, so a wait for a shared resource
9701    /// can be abandoned the moment the operator asks to park.
9702    pauses: Option<(&'a Pause, &'a Pause)>,
9703}
9704
9705/// Shared scheduling policy for operations and spawned seat invocations.
9706fn repository_wait(
9707    state: &RunState,
9708    pauses: Option<(&Pause, &Pause)>,
9709) -> crate::repo_lock::WaitPolicy {
9710    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9711    let inherited = crate::repo_lock::WaitPolicy::current();
9712    crate::repo_lock::WaitPolicy::new(
9713        Duration::from_secs(state.config.graph.timeout_resource_wait),
9714        move || {
9715            inherited.cancelled()
9716                || pauses
9717                    .as_ref()
9718                    .is_some_and(|(a, b)| a.parked() || b.parked())
9719        },
9720    )
9721}
9722
9723fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9724    if error.is::<crate::repo_lock::ResourceBlocked>() {
9725        state.resource_block = Some(crate::run::ResourceBlock {
9726            cause: crate::run::ResourceBlockCause::Other,
9727            node: "repository".into(),
9728            kind: if error.to_string().contains("disk pressure") {
9729                "disk pressure"
9730            } else {
9731                "repository coordination"
9732            }
9733            .into(),
9734            detail: error.to_string(),
9735            at: Timestamp::now(),
9736        });
9737        state.event("resource", error.to_string());
9738    }
9739}
9740
9741async fn coordinate_operation(
9742    state: &mut RunState,
9743    cwd: &Path,
9744    wait: &crate::repo_lock::WaitPolicy,
9745) -> Result<Option<crate::repo_lock::SeatLease>> {
9746    let result =
9747        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9748            .await;
9749    if let Err(e) = &result {
9750        record_repository_block(state, e);
9751    }
9752    result
9753}
9754
9755/// Run one job, honouring the parallelism budget.
9756async fn run_one(
9757    job: SeatJob,
9758    sem: Arc<Semaphore>,
9759    ctx: &WaveCtx<'_>,
9760    state: &mut RunState,
9761    attempt: usize,
9762) -> (SeatState, AgentOutcome) {
9763    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9764        .await
9765        .pop()
9766        .expect("one job in, one result out");
9767    (seat, out)
9768}
9769
9770/// Run every job concurrently, capped by the semaphore, preserving order.
9771///
9772/// Every seat in the batch is recorded into [`RunState::active`] before the
9773/// wave starts and cleared as each answer lands, so the run's own record says
9774/// who is still being waited on rather than only who finished.
9775async fn wave(
9776    jobs: Vec<SeatJob>,
9777    sem: Arc<Semaphore>,
9778    ctx: &WaveCtx<'_>,
9779    state: &mut RunState,
9780    attempt: usize,
9781) -> Vec<(usize, SeatState, AgentOutcome)> {
9782    let WaveCtx {
9783        run,
9784        node,
9785        prompts,
9786        cache,
9787        round,
9788        carry_seats: _,
9789        pauses,
9790    } = *ctx;
9791    for job in &jobs {
9792        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9793    }
9794    if let Err(e) = state.save() {
9795        // A failed persist of "who is answering right now" must not abort the
9796        // wave: the seats are already being asked, and the alternative is
9797        // losing the answers to save a status line nobody may even be
9798        // watching.
9799        tracing::warn!("could not persist in-progress seats: {e:#}");
9800    }
9801    // Hold the shared build cache's lease for the whole batch, not per job:
9802    // several candidates (an implement wave) or a fixer legitimately share
9803    // one cache concurrently within this run, and that stays untouched — a
9804    // single lease taken once for the whole wave and released once it is
9805    // done is what stops a *different* borrower (another run's own wave, its
9806    // e2e/gate, a human's `magi review`) from interleaving a build into the
9807    // same directory while this one is in flight. Best-effort, not
9808    // all-or-nothing: a wave that cannot get the lease within its own
9809    // longest job's budget still runs — an hour of paid implementer calls is
9810    // not thrown away over cache contention — but every write-allowed seat
9811    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9812    // below), the same fallback a read-only seat always gets, rather than
9813    // building into a directory this run was never granted. The identity
9814    // record is still invalidated below either way, so the next tracked
9815    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9816    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9817    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9818    // never borrowed from a job's agent budget (and never charged to it either,
9819    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9820    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9821    let cache_guard = if let Some(cache_dir) = cache {
9822        if jobs_had_a_writer {
9823            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9824            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9825            match acquire_cache_lease(
9826                state,
9827                cache_dir,
9828                &owner,
9829                resource_limit,
9830                node,
9831                node,
9832                &cancel,
9833            )
9834            .await
9835            {
9836                Ok(Some(g)) => Some(g),
9837                // Parking: leave without spawning anything. The run parks at
9838                // the next boundary and the seats are asked afresh on resume.
9839                Ok(None) => {
9840                    if let Some(block) = &mut state.resource_block {
9841                        block.cause = crate::run::ResourceBlockCause::Cancelled {
9842                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9843                        };
9844                    }
9845                    return block_wave(
9846                        jobs,
9847                        state,
9848                        node,
9849                        attempt,
9850                        "parked while waiting for the shared build cache",
9851                    );
9852                }
9853                // An unreadable lease is not contention: keep the old
9854                // fallback (no shared cache for this wave's seats).
9855                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9856                Err(e) => {
9857                    // Writing seats may not build into a cache this run does
9858                    // not hold, and an agent that cannot be given one is not
9859                    // asked at all: the run stops resumably instead.
9860                    let why = format!("{e:#}");
9861                    state.event(
9862                        "resource",
9863                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9864                    );
9865                    return block_wave(jobs, state, node, attempt, &why);
9866                }
9867            }
9868        } else {
9869            None
9870        }
9871    } else {
9872        None
9873    };
9874    // Every job keeps its whole budget: the wait above was not the agent's.
9875    // Restart the seats' clocks so "started" means "being spawned now".
9876    state.seats_spawning(node, attempt);
9877    if let Err(e) = state.save() {
9878        tracing::warn!("could not persist the start of a wave: {e:#}");
9879    }
9880    // Queueing behind this wave's own seats for a parallelism slot is normal
9881    // (each of them is bounded by its own agent timeout); only a wait beyond
9882    // all of them plus the resource limit means the slots never came back.
9883    let slot_limit = resource_limit
9884        + jobs
9885            .iter()
9886            .map(|j| j.timeout)
9887            .max()
9888            .unwrap_or_default()
9889            .saturating_mul(jobs.len() as u32);
9890    let mut set = tokio::task::JoinSet::new();
9891    // Seats announce themselves here the moment they hold a parallelism
9892    // permit, i.e. just before their CLI is spawned.
9893    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9894    let overlay = prompts.overlay(node);
9895    let repository_wait = repository_wait(state, pauses);
9896    for (i, mut job) in jobs.into_iter().enumerate() {
9897        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9898        if cache.is_some() {
9899            job.prompt.push('\n');
9900            job.prompt
9901                .push_str(&prompt::build_cache_note(node, job.allow_write));
9902        }
9903        let sem = Arc::clone(&sem);
9904        let run = run.to_owned();
9905        let node = node.to_owned();
9906        // Only implementers were told about the task's attachments, so only
9907        // their seats get the directory widened for reading.
9908        let attachments = if node == "implement" {
9909            state.attachments.clone()
9910        } else {
9911            Vec::new()
9912        };
9913        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9914        // `prompt::build_cache_note`'s doc for why setting it anyway is
9915        // exactly how a sandboxed reviewer's write refusal got reported as a
9916        // defect in the patch, not a property of its own seat. And a
9917        // write-allowed one is handed it only when the lease above was
9918        // actually acquired: a wave that could not get it (`cache_guard` is
9919        // `None`, see its own comment) must not send seats to build into a
9920        // directory this run does not hold - that is the exact concurrent,
9921        // unmanaged-write race this module exists to prevent, not something
9922        // "proceeding anyway" is allowed to reintroduce.
9923        let cache = cache
9924            .filter(|_| job.allow_write && cache_guard.is_some())
9925            .map(Path::to_path_buf);
9926        let spawned_tx = spawned_tx.clone();
9927        let repository_wait = repository_wait.clone();
9928        let coordination = state.config.coordination.clone();
9929        let fetched_base = state.fetched_base.clone();
9930        set.spawn(async move {
9931            // A wait for a parallelism slot is scheduling, not the agent: it
9932            // is bounded by its own limit and ends as a resource block, never
9933            // as a timeout of the agent that was not started.
9934            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9935                Ok(p) => p,
9936                Err(_) => {
9937                    return (
9938                        i,
9939                        job.seat,
9940                        AgentOutcome::ResourceBlocked(format!(
9941                            "no parallel slot became free within {}s",
9942                            slot_limit.as_secs()
9943                        )),
9944                        false,
9945                    );
9946                }
9947            };
9948            spawned_tx.send(job.seat.key.clone()).ok();
9949            let mut seat = job.seat;
9950            let out = repository_wait
9951                .scope(agent::invoke(
9952                    &job.spec,
9953                    &mut seat,
9954                    &Invocation {
9955                        coordination: coordination.as_ref(),
9956                        fetched_base: fetched_base.as_ref(),
9957                        cwd: &job.cwd,
9958                        prompt: &job.prompt,
9959                        timeout: job.timeout,
9960                        allow_write: job.allow_write,
9961                        sessions: job.sessions,
9962                        artifacts: &job.artifacts,
9963                        stem: &job.stem,
9964                        run: &run,
9965                        node: &node,
9966                        cache_dir: cache.as_deref(),
9967                        attachments: &attachments,
9968                        writable: &[],
9969                    },
9970                ))
9971                .await;
9972            let interrupted = out
9973                .as_ref()
9974                .err()
9975                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9976            let out = match out {
9977                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9978                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9979                // Billed work the CLI failed to hand over is not an ordinary
9980                // failure, but its text is the CLI's raw error JSON, not an
9981                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9982                // read it as one by forgetting to check. `usable()` is always
9983                // false here (dropped implies an empty response), so this has
9984                // to be checked before the catch-all `Failed` below or the
9985                // one shape this exists for is lost with the rest.
9986                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9987                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9988                Ok(o) => AgentOutcome::Failed(format!(
9989                    "process exited with no usable output ({})",
9990                    crate::notices::exit_status(o.exit_code)
9991                )),
9992                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9993                    AgentOutcome::ResourceBlocked(e.to_string())
9994                }
9995                Err(e) => AgentOutcome::Failed(e.to_string()),
9996            };
9997            (i, seat, out, interrupted)
9998        });
9999    }
10000    drop(spawned_tx);
10001    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10002    loop {
10003        let joined = tokio::select! {
10004            Some(key) = spawned_rx.recv() => {
10005                // The clock of this seat starts now, not when it queued.
10006                state.seat_spawned(&key);
10007                if let Err(e) = state.save() {
10008                    tracing::warn!("could not persist a seat's start: {e:#}");
10009                }
10010                continue;
10011            }
10012            joined = set.join_next() => joined,
10013        };
10014        let Some(joined) = joined else {
10015            break;
10016        };
10017        let (i, seat, out, interrupted) = match joined {
10018            Ok(v) => v,
10019            // No seat to clear: a panicked task never reported which one it
10020            // was. The defensive sweep below this loop is what stops that
10021            // seat's `active` entry from surviving forever.
10022            Err(e) => {
10023                tracing::error!("agent task panicked: {e}");
10024                continue;
10025            }
10026        };
10027        state.seat_finished(&seat.key);
10028        if interrupted
10029            && node == "implement"
10030            && let Some(label) = state
10031                .candidates
10032                .iter()
10033                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10034                .map(|candidate| candidate.label)
10035        {
10036            state.candidate_preservation.insert(
10037                label,
10038                "interrupted implementation awaiting persistence".into(),
10039            );
10040            state.interrupted_implementations.insert(label);
10041        }
10042        if let AgentOutcome::ResourceBlocked(why) = &out
10043            && state.resource_block.is_none()
10044        {
10045            state.resource_block = Some(crate::run::ResourceBlock {
10046                cause: crate::run::ResourceBlockCause::Other,
10047                node: node.to_owned(),
10048                kind: if why.contains("disk pressure") {
10049                    "disk pressure"
10050                } else if why.contains("repository coordination") {
10051                    "repository coordination"
10052                } else {
10053                    "parallel slot"
10054                }
10055                .to_owned(),
10056                detail: why.clone(),
10057                at: Timestamp::now(),
10058            });
10059            let activity = if interrupted {
10060                "agent execution interrupted"
10061            } else {
10062                "no agent was started"
10063            };
10064            state.event("resource", format!("{node}: {activity} - {why}"));
10065        }
10066        record_jobs(state, node, round, &seat.key, &out);
10067        if let Err(e) = state.save() {
10068            tracing::warn!("could not persist a seat's completion: {e:#}");
10069        }
10070        if collected.len() <= i {
10071            collected.resize_with(i + 1, || None);
10072        }
10073        collected[i] = Some((i, seat, out));
10074    }
10075    // Belt-and-braces for the panic branch above: every seat this exact batch
10076    // started shares this `(node, attempt)` pair, and every seat that finished
10077    // normally already cleared itself, so anything left tagged with it here
10078    // can only be a panicked task's leftover. Cleared unconditionally rather
10079    // than left to read as still answering forever.
10080    if state
10081        .active
10082        .values()
10083        .any(|a| a.node == node && a.attempt == attempt)
10084    {
10085        state
10086            .active
10087            .retain(|_, a| !(a.node == node && a.attempt == attempt));
10088        if let Err(e) = state.save() {
10089            tracing::warn!("could not persist the end of a wave: {e:#}");
10090        }
10091    }
10092    // Whether or not the lease above was actually held, several worktrees
10093    // may just have built into the cache with nothing here able to name one
10094    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
10095    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
10096    // clean it might not have strictly needed; trusting a stale match would
10097    // cost it a wrong answer.
10098    if let Some(cache_dir) = cache
10099        && jobs_had_a_writer
10100    {
10101        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10102    }
10103    if let Some(guard) = cache_guard {
10104        guard.release();
10105    }
10106    collected.into_iter().flatten().collect()
10107}
10108
10109/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
10110/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
10111/// this is data collection, not the fix-specific completion contract in
10112/// [`Runner::continue_fix_report`], and applies regardless of which node
10113/// asked.
10114///
10115/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
10116/// evidence from; `Failed` does not, and correctly contributes nothing — a
10117/// timeout or crash is not itself evidence about a command the seat may have
10118/// started.
10119fn record_jobs(
10120    state: &mut RunState,
10121    node: &str,
10122    round: Option<usize>,
10123    seat: &str,
10124    out: &AgentOutcome,
10125) {
10126    let commands: &[agent::CommandEvidence] = match out {
10127        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10128        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10129    };
10130    let checked_at = Timestamp::now();
10131    for c in commands {
10132        state.jobs.push(JobRecord {
10133            node: node.to_owned(),
10134            round,
10135            seat: seat.to_owned(),
10136            id: c.id.clone(),
10137            description: c.description.clone(),
10138            checked_at,
10139            status: match c.exit_code {
10140                Some(0) => JobStatus::Completed,
10141                Some(_) => JobStatus::Failed,
10142                None => JobStatus::Unknown,
10143            },
10144            exit_code: c.exit_code,
10145            result_summary: c.result_summary.clone(),
10146            source: c.source.clone(),
10147        });
10148    }
10149}
10150
10151/// Is a review round clean, given how many reviewer seats answered against
10152/// how many the round expected?
10153///
10154/// A seat that never answered (timeout, crash, unparsable output) is not a
10155/// seat that read the patch and found nothing — treating it as such is
10156/// exactly the bug this function exists to close. Under the default `block`
10157/// policy a missing seat can never be clean; `warn` still requires the seats
10158/// that *did* answer to have found nothing blocking and verification to be
10159/// green.
10160///
10161/// `quota_missing` narrows that `block` default for exactly one cause of
10162/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
10163/// a session limit lifts by the very next round buys nothing — the seat is
10164/// asked again with the same quota — so once every missing seat is accounted
10165/// for by a quota loss (and at least one seat *did* answer, so a decision has
10166/// something to rest on) the round is decided on the panel that could answer,
10167/// same as `warn` would. A panel that lost every seat to quota is not
10168/// decided here: `answered == 0` falls through to the existing `block`
10169/// fallback so a fully collapsed panel still waits rather than landing on no
10170/// review at all.
10171fn round_is_clean(
10172    blocking: usize,
10173    e2e_ok: bool,
10174    answered: usize,
10175    expected: usize,
10176    quota_missing: usize,
10177    policy: IncompleteReviewPolicy,
10178) -> bool {
10179    if blocking != 0 || !e2e_ok {
10180        return false;
10181    }
10182    if answered == expected || policy == IncompleteReviewPolicy::Warn {
10183        return true;
10184    }
10185    answered > 0 && expected - answered <= quota_missing
10186}
10187
10188/// The review loop's own conclusion, derived entirely from its persisted
10189/// round records and the round budget that produced them — never from
10190/// `status`, so a reentry (or `gate`/`merge` reading it independently)
10191/// recomputes the identical answer regardless of what an earlier node in the
10192/// same walk, or a previous walk, did to `status`.
10193///
10194/// `None` while more rounds remain to try, including when review never ran
10195/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
10196/// gone clean, or the budget is spent, or the tree has stopped moving (see
10197/// [`STAGNANT_LIMIT`]), the answer is one of two things:
10198///
10199/// - An incomplete panel that raised nothing is missing input, not a
10200///   verified tree — never a hand-off candidate, whatever verification said
10201///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
10202/// - Otherwise, green e2e on the last round hands off (see
10203///   [`Runner::stop_reviewing`]); red e2e blocks.
10204///
10205/// A last round whose own verification is still `ResourceBlocked` — magi
10206/// itself never got a command to run, not evidence the patch is broken —
10207/// is neither: this returns `None` for it too, the same as "more rounds
10208/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
10209/// handling of that shape) instead of this cheap recomputation guessing a
10210/// verdict a real attempt never produced.
10211pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10212    if max_rounds == 0
10213        || reviews.last().is_some_and(|r| {
10214            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10215        })
10216    {
10217        return Some(RunStatus::Gating);
10218    }
10219    let last = reviews.last()?;
10220    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10221    if reviews.len() < max_rounds && !stagnant {
10222        return None;
10223    }
10224    if last.incomplete() && last.blocking == 0 {
10225        return Some(RunStatus::Blocked);
10226    }
10227    if last.e2e_status() == E2eStatus::ResourceBlocked
10228        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10229    {
10230        return None;
10231    }
10232    Some(if last.e2e.iter().all(CommandOutcome::ok) {
10233        RunStatus::Gating
10234    } else {
10235        RunStatus::Blocked
10236    })
10237}
10238
10239/// How long a re-ask may take, given the budget the first attempt had.
10240///
10241/// A `nudged` retry is a request to restate an answer the seat has already
10242/// worked out: it carries no new work, so it does not deserve the original
10243/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
10244/// 133 seconds while a third sat for over ten minutes on a resumed session
10245/// holding 410 KB of prior output - and because the retry had inherited the
10246/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
10247/// judging round whose other seats were long finished.
10248///
10249/// A quarter of the budget, with a floor so that a deliberately short timeout
10250/// does not collapse to nothing. A retry that re-sends the whole prompt
10251/// (because the seat kept no context) is the original job again, and keeps the
10252/// original budget.
10253pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10254    if nudged {
10255        (full / 4).max(Duration::from_secs(120)).min(full)
10256    } else {
10257        full
10258    }
10259}
10260
10261/// Run a wave and parse each reply, re-asking the seats whose reply was
10262/// unusable.
10263///
10264/// The re-ask is a nudge rather than the whole prompt again when the seat still
10265/// holds its conversation, which is the difference between a cheap retry and
10266/// paying for the entire candidate set twice.
10267///
10268/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
10269/// successor in `roster` is handed to it instead of being re-asked: the
10270/// handover is the retry. A seat with no successor left (a single-agent
10271/// roster, the roster's tail) is nudged as before, up to `retries` times. So
10272/// the asks to one seat in one node number at most
10273/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
10274/// is asked once (a dropped stream is nudged first), and only the last agent
10275/// of the chain gets the `retries` same-agent nudges. Each roster agent is
10276/// tried at most once per seat, walking forward from the seat's own position and never wrapping
10277/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
10278/// other failure stops the chain when the previous agent failed the same way
10279/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
10280/// [`has_context`] is false and the job's own full prompt and full budget are
10281/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
10282/// intermediate ones are not losses) and is returned as a failure like any
10283/// other absent seat — the caller decides whether the panel still has a
10284/// quorum. An empty `roster` disables handover: failures are nudged as they
10285/// always were, and a quota is simply lost. A reply that fails to parse or
10286/// validate is the prompt's doing and is only ever nudged, never handed over.
10287///
10288/// The returned [`SeatState`] names the agent that answered (or tried last).
10289#[allow(clippy::too_many_arguments)]
10290async fn ask_json_wave<T>(
10291    jobs: Vec<SeatJob>,
10292    sem: Arc<Semaphore>,
10293    retries: usize,
10294    roster: &[AgentSpec],
10295    ctx: &WaveCtx<'_>,
10296    losses: &mut Vec<QuotaLoss>,
10297    state: &mut RunState,
10298    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10299) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10300where
10301    T: serde::de::DeserializeOwned + Send + 'static,
10302{
10303    ask_wave_with(
10304        jobs,
10305        sem,
10306        retries,
10307        roster,
10308        ctx,
10309        losses,
10310        state,
10311        &|text: &str| {
10312            let v = verdict::extract_json::<T>(text)?;
10313            validate(&v)?;
10314            Ok(v)
10315        },
10316    )
10317    .await
10318}
10319
10320/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
10321/// a node whose answer is prose (deliberation) shares the same handover,
10322/// failure classification, quota bookkeeping and bounds instead of a copy.
10323///
10324/// A seat handed to another roster agent is sent the job's `handover` prompt
10325/// (when it has one) rather than `prompt`: the new agent has no session, so a
10326/// resume-style prompt would be incomplete. That holds for the handover ask
10327/// and for every nudge to that agent whose `has_context` is false.
10328#[allow(clippy::too_many_arguments)]
10329async fn ask_wave_with<T>(
10330    jobs: Vec<SeatJob>,
10331    sem: Arc<Semaphore>,
10332    retries: usize,
10333    roster: &[AgentSpec],
10334    ctx: &WaveCtx<'_>,
10335    losses: &mut Vec<QuotaLoss>,
10336    state: &mut RunState,
10337    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10338) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10339where
10340    T: Send + 'static,
10341{
10342    let n = jobs.len();
10343    let originals: Vec<SeatJob> = jobs;
10344    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10345    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10346    // Nudges each seat's *current* agent has taken — 0 for a first-ask
10347    // answer, N once it has gone through N nudges. Read back once this
10348    // returns, so a caller building a history record (`ReviewRecord`) can
10349    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
10350    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
10351    // field's own doc.
10352    let mut nudges: Vec<usize> = vec![0; n];
10353    // The agent now occupying each seat, the ids it has already been through,
10354    // where in the roster the walk began, the class of the last failure, and
10355    // the stem word of a handover not yet asked (full prompt, full budget).
10356    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10357    let mut tried: Vec<BTreeSet<String>> = specs
10358        .iter()
10359        .map(|s| BTreeSet::from([s.id.clone()]))
10360        .collect();
10361    let starts: Vec<usize> = specs
10362        .iter()
10363        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10364        .collect();
10365    let carry = ctx.carry_seats && !roster.is_empty();
10366    // Carried across rounds: ids that failed the seat earlier, and how the
10367    // last failure went (so a repeat of it is not handed over again).
10368    let carried: Vec<BTreeSet<String>> = originals
10369        .iter()
10370        .map(|j| {
10371            state
10372                .seat_history
10373                .get(&j.seat.key)
10374                .filter(|_| carry)
10375                .map(|h| h.failed.clone())
10376                .unwrap_or_default()
10377        })
10378        .collect();
10379    let mut prev: Vec<Option<FailClass>> = originals
10380        .iter()
10381        .map(|j| {
10382            state
10383                .seat_history
10384                .get(&j.seat.key)
10385                .filter(|_| carry)
10386                .and_then(|h| h.last_fail.clone())
10387        })
10388        .collect();
10389    let next_agent =
10390        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10391            let others: BTreeSet<String> = specs
10392                .iter()
10393                .enumerate()
10394                .filter(|(j, _)| *j != i)
10395                .map(|(_, s)| s.id.clone())
10396                .collect();
10397            pick_successor(
10398                roster,
10399                starts[i],
10400                tried,
10401                carry.then(|| &carried[i]),
10402                &others,
10403            )
10404            .cloned()
10405        };
10406    let mut fresh: Vec<Option<String>> = vec![None; n];
10407    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10408    let mut pending: Vec<usize> = (0..n).collect();
10409
10410    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
10411    // asks: an agent with a successor is asked once and handed over, and only
10412    // a seat with no successor spends `retries` nudges on the same agent. This
10413    // only guarantees the loop's own termination whatever those say.
10414    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10415    for round in 0..max_rounds {
10416        if pending.is_empty() {
10417            break;
10418        }
10419        let mut batch = Vec::with_capacity(pending.len());
10420        let mut renudged: Vec<&str> = Vec::new();
10421        for &i in &pending {
10422            let src = &originals[i];
10423            // A seat now held by another agent than the job named has no
10424            // session of its own: it gets the full-context prompt whenever
10425            // it is asked in full (the handover ask, a nudge it cannot
10426            // resume).
10427            let full: &str = match &src.handover {
10428                Some(h) if specs[i].id != src.spec.id => h,
10429                _ => &src.prompt,
10430            };
10431            // The prompt and the budget are one decision: a nudge restates
10432            // finished work, a re-sent prompt redoes it.
10433            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10434                (
10435                    full.to_owned(),
10436                    src.timeout,
10437                    format!("{}-{word}-{}", src.stem, specs[i].id),
10438                )
10439            } else if nudges[i] == 0 {
10440                (full.to_owned(), src.timeout, src.stem.clone())
10441            } else {
10442                renudged.push(src.seat.key.as_str());
10443                let why = done[i]
10444                    .as_ref()
10445                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
10446                    .unwrap_or_else(|| "no parsable answer".to_owned());
10447                let nudge = prompt::nudge(&why);
10448                let nudged = has_context(&specs[i], &seats[i], src.sessions);
10449                let prompt = if nudged {
10450                    nudge
10451                } else {
10452                    format!("{full}\n\n---\n\n{nudge}")
10453                };
10454                (
10455                    prompt,
10456                    retry_budget(src.timeout, nudged),
10457                    format!("{}-retry{}", src.stem, nudges[i]),
10458                )
10459            };
10460            batch.push(SeatJob {
10461                spec: specs[i].clone(),
10462                seat: seats[i].clone(),
10463                cwd: src.cwd.clone(),
10464                prompt,
10465                timeout,
10466                allow_write: src.allow_write,
10467                sessions: src.sessions,
10468                artifacts: src.artifacts.clone(),
10469                stem,
10470                handover: None,
10471            });
10472        }
10473
10474        if !renudged.is_empty() {
10475            state.event(
10476                ctx.node,
10477                format!("retry {round}: re-asking {}", renudged.join(", ")),
10478            );
10479        }
10480        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10481        let mut still = Vec::new();
10482        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10483            seats[i] = seat;
10484            // No agent was asked: nothing to hand over, nudge, remember or
10485            // charge. The caller sees `resource_block` and stops the node.
10486            if let AgentOutcome::ResourceBlocked(why) = &out {
10487                done[i] = Some(Err(anyhow::anyhow!("{why}")));
10488                continue;
10489            }
10490            let class = FailClass::of(&out);
10491            // A dropped stream is nudged first (the conversation is still
10492            // there to pick up); only a seat whose nudges are spent hands over.
10493            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10494                && nudges[i] < retries
10495                && !roster.is_empty();
10496            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10497                let next = should_hand_over(prev[i].as_ref(), &cur)
10498                    .then(|| next_agent(i, &tried[i], &specs))
10499                    .flatten();
10500                if carry {
10501                    let h = state
10502                        .seat_history
10503                        .entry(originals[i].seat.key.clone())
10504                        .or_default();
10505                    h.failed.insert(specs[i].id.clone());
10506                    h.last_fail = Some(cur.clone());
10507                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10508                        h.last_ok = None;
10509                    }
10510                    // Saved before the next agent is asked, so a restart in
10511                    // between does not forget who failed.
10512                    if let Err(e) = state.save() {
10513                        tracing::warn!("could not persist a seat's failure history: {e:#}");
10514                    }
10515                }
10516                if let Some(next) = next {
10517                    record_handover(
10518                        state,
10519                        ctx.node,
10520                        &originals[i].seat.key,
10521                        &specs[i].id,
10522                        &next.id,
10523                        &cur,
10524                        &fail_reason(&out),
10525                    );
10526                    tried[i].insert(next.id.clone());
10527                    prev[i] = Some(cur.clone());
10528                    seats[i] =
10529                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10530                    specs[i] = next;
10531                    fresh[i] = Some(cur.stem_word().to_owned());
10532                    nudges[i] = 0;
10533                    done[i] = Some(Err(anyhow::anyhow!(
10534                        "handed over after: {}",
10535                        fail_reason(&out)
10536                    )));
10537                    still.push(i);
10538                    continue;
10539                }
10540            }
10541            if carry
10542                && !nudge_first
10543                && let Some(cur) = class.clone()
10544            {
10545                // The chain ended here (no successor, or a repeated failure).
10546                let h = state
10547                    .seat_history
10548                    .entry(originals[i].seat.key.clone())
10549                    .or_default();
10550                h.failed.insert(specs[i].id.clone());
10551                h.last_fail = Some(cur);
10552            }
10553            let parsed = match out {
10554                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10555                AgentOutcome::Quota(o) => {
10556                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10557                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10558                }
10559                // Not a parseable answer: the nudge loop re-asks it, which is
10560                // exactly what a dropped stream needs. Just don't hand its raw
10561                // error JSON to `extract_json`.
10562                AgentOutcome::Dropped(o) => {
10563                    let why = o
10564                        .dropped
10565                        .as_ref()
10566                        .map(|d| d.why.as_str())
10567                        .unwrap_or("the CLI ended the stream without delivering its answer");
10568                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10569                }
10570                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10571                    Err(anyhow::anyhow!(e))
10572                }
10573            };
10574            let quota = class == Some(FailClass::Quota);
10575            let failed = parsed.is_err();
10576            done[i] = Some(parsed);
10577            if carry && !failed {
10578                let h = state
10579                    .seat_history
10580                    .entry(originals[i].seat.key.clone())
10581                    .or_default();
10582                h.failed.remove(&specs[i].id);
10583                h.last_ok = Some(specs[i].id.clone());
10584                h.last_fail = None;
10585            }
10586            // Do not re-ask a rate-limited seat (quota) — a retry is known to
10587            // fail the same way; and never re-ask a seat that already parsed.
10588            // A failed agent that still has a successor is not re-asked
10589            // either: the handover was its remedy and has just been refused
10590            // (the chain stops on a repeated failure class). A seat with no
10591            // successor left (a single-agent roster, the roster's tail, or an
10592            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10593            let agent_failure = class.is_some()
10594                && !nudge_first
10595                && !roster.is_empty()
10596                && next_agent(i, &tried[i], &specs).is_some();
10597            if failed && !quota && !agent_failure && nudges[i] < retries {
10598                nudges[i] += 1;
10599                still.push(i);
10600            }
10601        }
10602        pending = still;
10603    }
10604
10605    // One loss per seat whose chain ended on a quota: the intermediate ones
10606    // were absorbed by a handover and are not losses.
10607    for (i, q) in last_quota.into_iter().enumerate() {
10608        if let Some(reset) = q {
10609            losses.push(QuotaLoss {
10610                seat: originals[i].seat.key.clone(),
10611                node: ctx.node.to_owned(),
10612                at: Timestamp::now(),
10613                reset,
10614            });
10615        }
10616    }
10617
10618    seats
10619        .into_iter()
10620        .zip(done)
10621        .zip(nudges)
10622        .map(|((seat, res), attempts)| {
10623            (
10624                seat,
10625                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10626                attempts,
10627            )
10628        })
10629        .collect()
10630}
10631
10632/// Did the wave end without asking any agent (see [`block_wave`])?
10633fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10634    results
10635        .iter()
10636        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10637}
10638
10639/// Settle a wave that never spawned anything: every job answers
10640/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10641/// else about the seat (session, history, quota) is touched.
10642fn block_wave(
10643    jobs: Vec<SeatJob>,
10644    state: &mut RunState,
10645    node: &str,
10646    attempt: usize,
10647    why: &str,
10648) -> Vec<(usize, SeatState, AgentOutcome)> {
10649    state.resource_wait = None;
10650    state
10651        .active
10652        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10653    if let Err(e) = state.save() {
10654        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10655    }
10656    jobs.into_iter()
10657        .enumerate()
10658        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10659        .collect()
10660}
10661
10662/// Acquire the shared build cache's lease, waiting out contention within
10663/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10664/// unbounded wait is never acceptable).
10665///
10666/// A first, non-blocking check happens before ever waiting; if it finds the
10667/// lease busy, that fact is logged as a `verify` event *and* flushed with
10668/// [`RunState::save`] immediately — not only once the wait finally succeeds
10669/// or gives up — so a `magi show` run by a different process while this one
10670/// is still waiting reads a `run.json` that says so, rather than whatever it
10671/// looked like before the wait started. The same applies to the terminal
10672/// failure: logged and saved before this returns `Err`, so a caller that
10673/// could not get the lease at all still leaves a legible record of why.
10674async fn acquire_cache_lease(
10675    state: &mut RunState,
10676    cache_dir: &Path,
10677    owner: &crate::cache::Owner,
10678    budget: Duration,
10679    node: &str,
10680    context: &str,
10681    cancel: &(dyn Fn() -> bool + Sync),
10682) -> Result<Option<crate::cache::Guard>> {
10683    let home = crate::run::home();
10684    let started = Instant::now();
10685    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10686        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10687        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10688        Err(e) => {
10689            state.event(
10690                "verify",
10691                format!("{context}: could not check the shared build cache: {e:#}"),
10692            );
10693            if let Err(e2) = state.save() {
10694                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10695            }
10696            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10697        }
10698    };
10699    state.event(
10700        "verify",
10701        format!(
10702            "{context}: waiting for the shared build cache at {} ({})",
10703            cache_dir.display(),
10704            busy.describe()
10705        ),
10706    );
10707    // Seats of this node are waiting, not running: say so, so the countdown a
10708    // reader sees only ever measures an agent that has been spawned.
10709    state.resource_wait = Some(crate::run::ResourceWait {
10710        node: context.to_owned(),
10711        kind: "shared build cache".to_owned(),
10712        holder: busy.describe(),
10713        since: Timestamp::now(),
10714        limit_secs: budget.as_secs(),
10715    });
10716    if let Err(e) = state.save() {
10717        tracing::warn!("could not persist a cache wait: {e:#}");
10718    }
10719    let remaining = budget.saturating_sub(started.elapsed());
10720    let waited = crate::cache::wait_for_unless(
10721        &home,
10722        cache_dir,
10723        owner,
10724        remaining,
10725        Duration::from_secs(5),
10726        cancel,
10727    )
10728    .await;
10729    state.resource_wait = None;
10730    let cause = match &waited {
10731        Ok(Some(_)) => None,
10732        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10733        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10734            Some(crate::run::ResourceBlockCause::Timeout {
10735                waited_secs: started.elapsed().as_secs(),
10736                limit_secs: budget.as_secs(),
10737            })
10738        }
10739        Err(_) => Some(crate::run::ResourceBlockCause::Other),
10740    };
10741    if let Some(cause) = cause {
10742        let detail = match &waited {
10743            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10744            Err(error) => format!("{error:#}"),
10745        };
10746        state.resource_block = Some(crate::run::ResourceBlock {
10747            cause,
10748            node: node.to_owned(),
10749            kind: "shared build cache".into(),
10750            detail,
10751            at: Timestamp::now(),
10752        });
10753        state.event(
10754            "resource",
10755            state.resource_block.as_ref().unwrap().description(),
10756        );
10757    }
10758    if let Err(error) = state.save() {
10759        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10760    }
10761    waited
10762}
10763
10764/// The lease state could not be read at all (unreadable lease file, a
10765/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10766/// known to be contended, so a wave degrades to running without a shared cache
10767/// instead of stopping on a hold that [`crate::triage`] could never release.
10768#[derive(Debug)]
10769struct CacheCheckError(String);
10770
10771impl std::fmt::Display for CacheCheckError {
10772    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10773        f.write_str(&self.0)
10774    }
10775}
10776
10777impl std::error::Error for CacheCheckError {}
10778
10779/// Keep a specific acquisition outcome when the caller adds command evidence.
10780fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10781    state
10782        .resource_block
10783        .get_or_insert_with(|| crate::run::ResourceBlock {
10784            cause: crate::run::ResourceBlockCause::Other,
10785            node: node.to_owned(),
10786            kind: "shared build cache".into(),
10787            detail: format!("{error:#}"),
10788            at: Timestamp::now(),
10789        });
10790    if let Err(error) = state.save() {
10791        tracing::warn!("could not persist cache resource block: {error:#}");
10792    }
10793}
10794
10795/// Run `body` — a verify command batch — while holding the shared build
10796/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10797/// never interleave with another borrower's build against the same
10798/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10799/// timeout, or a human's own `magi review`. See the `cache` module doc for
10800/// why this matters more than Cargo's own per-target locking covers — two
10801/// *different* worktrees building the same package name/version into one
10802/// cache directory is a staleness bug, not a lock contention one.
10803///
10804/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10805/// agent waves do. Repository scheduling also has its own bounded,
10806/// cancellable resource-wait policy. Contention must not consume the
10807/// verification budget: `body` receives the full `budget` after both leases
10808/// are acquired and the cache is fresh. Without a shared cache, only
10809/// repository coordination is acquired before handing over the full budget.
10810///
10811/// A lease that cannot be acquired within the resource wait is reported as a single
10812/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10813/// verification — the same shape a spawn failure already takes in
10814/// [`run_commands`], so a caller need not special-case it.
10815#[allow(clippy::too_many_arguments)]
10816async fn with_cache_lease<'s, F, Fut>(
10817    state: &'s mut RunState,
10818    cache_dir: Option<&Path>,
10819    node: &str,
10820    seat: &str,
10821    worktree: &Path,
10822    head: &str,
10823    budget: Duration,
10824    context: &str,
10825    wait: &crate::repo_lock::WaitPolicy,
10826    body: F,
10827) -> Result<(Vec<CommandOutcome>, bool)>
10828where
10829    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10830    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10831{
10832    // Match editing waves: cache first, repository second. In particular,
10833    // pre_gate must never hold the repository while waiting for a wave's cache.
10834    let started = Instant::now();
10835    let home = crate::run::home();
10836    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10837    let guard = if let Some(cache_dir) = cache_dir {
10838        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10839        match acquire_cache_lease(
10840            state,
10841            cache_dir,
10842            &owner,
10843            resource_limit,
10844            node,
10845            context,
10846            &|| wait.cancelled(),
10847        )
10848        .await
10849        .and_then(|g| {
10850            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10851        }) {
10852            Ok(g) => Some(g),
10853            Err(e) => {
10854                record_cache_error(state, node, &e);
10855                return Ok((
10856                    vec![CommandOutcome {
10857                        command: "(waiting for the shared build cache)".into(),
10858                        code: None,
10859                        output_tail: e.to_string(),
10860                        duration_ms: started.elapsed().as_millis() as u64,
10861                        resource_blocked: true,
10862                    }],
10863                    false,
10864                ));
10865            }
10866        }
10867    } else {
10868        None
10869    };
10870    let _coordination = coordinate_operation(state, worktree, wait).await?;
10871    if let Some(cache_dir) = cache_dir {
10872        let identity = crate::cache::Identity::new(worktree, head);
10873        if let Err(e) = crate::cache::ensure_fresh(
10874            &home,
10875            cache_dir,
10876            &identity,
10877            &state.config.verify.cargo_manifests(),
10878        ) {
10879            record_cache_error(state, node, &e);
10880            state.event(
10881                "verify",
10882                format!(
10883                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10884                    worktree.display(),
10885                    short(head)
10886                ),
10887            );
10888            if let Some(guard) = guard {
10889                guard.release();
10890            }
10891            return Ok((
10892                vec![CommandOutcome {
10893                    command: "(confirming the shared build cache is fresh)".into(),
10894                    code: None,
10895                    output_tail: e.to_string(),
10896                    duration_ms: started.elapsed().as_millis() as u64,
10897                    resource_blocked: true,
10898                }],
10899                false,
10900            ));
10901        }
10902    }
10903    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10904    if !timed_out_pids.is_empty() {
10905        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10906    }
10907    // Drop repository coordination before releasing the cache, reversing the
10908    // acquisition order. A cancelled/error future also drops both guards.
10909    drop(_coordination);
10910    if let Some(guard) = guard {
10911        guard.release();
10912    }
10913    Ok((outcomes, retried))
10914}
10915
10916/// Poll `pids` — commands [`run_commands`] reports as still running when its
10917/// own timeout elapsed — until every one is confirmed gone, or
10918/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10919///
10920/// Real confirmation where confirmation is possible, not a substitute for
10921/// full process-tree observation: a grandchild the timed-out process spawned
10922/// and that survives independently of it is invisible to a pid check the
10923/// same way it always was, and continuing to observe and collect *that*
10924/// stays a different piece of work with its own owner. This only narrows a
10925/// fixed blind wait into an actual check of the pids this process does know
10926/// about.
10927async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10928    wait_for_pids_with(
10929        pids,
10930        crate::proc::pid_alive,
10931        LEASE_RELEASE_POLL,
10932        LEASE_RELEASE_MAX_WAIT,
10933    )
10934    .await;
10935}
10936
10937/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10938/// interval and ceiling supplied by the caller, so the polling *logic* -
10939/// returns as soon as every pid reports dead, gives up at the ceiling
10940/// otherwise - is testable on millisecond durations without asking the real
10941/// OS about a pid at all.
10942async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10943    pids: &[u32],
10944    alive: F,
10945    poll: Duration,
10946    max_wait: Duration,
10947) {
10948    let deadline = Instant::now() + max_wait;
10949    loop {
10950        if pids.iter().all(|&pid| !alive(pid)) {
10951            return;
10952        }
10953        if Instant::now() >= deadline {
10954            return;
10955        }
10956        tokio::time::sleep(poll).await;
10957    }
10958}
10959
10960/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10961/// admission that it could not even get a verify command to run, as opposed
10962/// to evidence the command actually produced? A caller that would otherwise
10963/// read a resource-blocked outcome as a red command must check this first:
10964/// see [`Runner::gate`], which retries rather than records `Blocked` when
10965/// this is true.
10966fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10967    outcomes.iter().any(|o| o.resource_blocked)
10968}
10969
10970/// What [`Runner::gate_fix_round`] decided.
10971pub(crate) enum GateFix {
10972    /// The tree changed and `verify.e2e` is still green: run the gate again.
10973    Retry,
10974    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10975    /// last failure stands and the run ends blocked.
10976    Stop,
10977    /// `verify.e2e` could not run after the fix (magi's own contention):
10978    /// decide nothing now, a later reentry retries.
10979    Defer,
10980}
10981
10982/// Is every red command in `outcomes` an ordinary failure the code could
10983/// explain: it ran, exited non-zero, and said something?
10984///
10985/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10986/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10987/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10988/// what the command is stays the gate's business.
10989fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10990    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10991    red.peek().is_some()
10992        && red.all(|o| {
10993            !o.resource_blocked
10994                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10995                && !o.output_tail.trim().is_empty()
10996        })
10997}
10998
10999/// Describe one verify command's outcome for the event log, distinguishing a
11000/// build/link failure — the toolchain never produced a binary to run — from
11001/// an actual test failure, since only the latter is a verdict on the patch.
11002fn e2e_outcome_label(o: &CommandOutcome) -> String {
11003    if o.ok() {
11004        return "pass".to_owned();
11005    }
11006    let reason = if o.build_failed() {
11007        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11008    } else {
11009        format!("FAIL ({:?})", o.code)
11010    };
11011    format!(
11012        "{reason}\n{}",
11013        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11014    )
11015}
11016
11017/// Run `verify.e2e`, retrying once if the first attempt could not build or
11018/// link — a build/link failure is frequently a race against a shared
11019/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
11020/// `verify` event per command, tagged with `context` (normally `"round N"`)
11021/// so the two call sites that need this — the ordinary per-round leg in
11022/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
11023/// it will ever call a round green — read identically in the event log.
11024async fn run_e2e_with_retry(
11025    state: &mut RunState,
11026    shell: &[String],
11027    commands: &[String],
11028    worktree: &Path,
11029    timeout: Duration,
11030    context: &str,
11031) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11032    let (mut e2e, mut timed_out_pids) = run_commands(
11033        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11034    )
11035    .await;
11036    for o in &e2e {
11037        state.event(
11038            "verify",
11039            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11040        );
11041    }
11042    // A build/link failure is not a verdict on the patch — it is frequently a
11043    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
11044    // one retry before letting a red like that decide the round.
11045    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11046    if verify_retried {
11047        state.event(
11048            "verify",
11049            format!(
11050                "{context}: verify could not build/link, not a test result — retrying once \
11051                 before concluding"
11052            ),
11053        );
11054        let retried = run_commands(
11055            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11056        )
11057        .await;
11058        e2e = retried.0;
11059        // Both attempts' timeouts matter, not just the last one: the first
11060        // attempt's descendants may still be alive alongside the retry's.
11061        timed_out_pids.extend(retried.1);
11062        for o in &e2e {
11063            state.event(
11064                "verify",
11065                format!(
11066                    "{context}: retry `{}` -> {}",
11067                    o.command,
11068                    e2e_outcome_label(o)
11069                ),
11070            );
11071        }
11072    }
11073    (e2e, verify_retried, timed_out_pids)
11074}
11075
11076/// Run configured shell commands in `cwd`, in order. The second element is
11077/// the pid of every command that hit `timeout` and was still running when
11078/// this stopped waiting on it (best-effort: `None` when the platform did not
11079/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
11080/// that releases a shared resource afterward needs to know.
11081///
11082/// Records `task` into [`RunState::active`] at every command boundary
11083/// (`RunState::task_command`) and clears it once the whole list has run
11084/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
11085/// for minutes with no seat and no output of its own to show for it (see
11086/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
11087/// apart from "not yet run" without this), and this is the only place that
11088/// knows which command is running right now and how many are left. Three
11089/// saves per command — start, not per second — matching the same "only at a
11090/// boundary" rule [`wave`] already follows for seats.
11091#[allow(clippy::too_many_arguments)]
11092async fn run_commands(
11093    state: &mut RunState,
11094    node: &str,
11095    task: &str,
11096    attempt: usize,
11097    shell: &[String],
11098    commands: &[String],
11099    cwd: &Path,
11100    timeout: Duration,
11101) -> (Vec<CommandOutcome>, Vec<u32>) {
11102    if commands.is_empty() {
11103        // Nothing to mark as running and nothing to clear — an empty list
11104        // means "not configured", and touching `active` (or the disk) over
11105        // that would be a write for every round of a repo with no
11106        // `verify.e2e` / `verify.gate` commands at all.
11107        return (Vec::new(), Vec::new());
11108    }
11109    let mut out = Vec::new();
11110    let mut timed_out_pids = Vec::new();
11111    let total = commands.len();
11112    for (idx, command) in commands.iter().enumerate() {
11113        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11114        if let Err(e) = state.save() {
11115            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11116        }
11117        let started = Instant::now();
11118        let mut cmd = tokio::process::Command::new(&shell[0]);
11119        cmd.quiet();
11120        // Verify the checkout as a standalone caller. Worker ownership and
11121        // pinned executable selection belong to this driver, not to tests or
11122        // tools it runs. Preserve home/seat context and Running's containment.
11123        for key in crate::worker::INVOCATION_ENV {
11124            cmd.env_remove(key);
11125        }
11126        cmd.args(&shell[1..])
11127            .arg(command)
11128            .current_dir(cwd)
11129            .stdin(std::process::Stdio::null())
11130            .stdout(std::process::Stdio::piped())
11131            .stderr(std::process::Stdio::piped())
11132            .kill_on_drop(true);
11133        let mut outputs = Vec::new();
11134        let protected = (|| -> Result<()> {
11135            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
11136                outputs.push(guard);
11137            }
11138            if let Some(cache) = state.config.cache_dir() {
11139                outputs.push(crate::retention::OutputLease::cache(
11140                    &crate::run::home(),
11141                    &cache,
11142                    false,
11143                )?);
11144            }
11145            Ok(())
11146        })();
11147        if let Err(e) = protected {
11148            out.push(CommandOutcome {
11149                command: command.clone(),
11150                code: None,
11151                output_tail: format!("output ownership unavailable: {e:#}"),
11152                duration_ms: 0,
11153                resource_blocked: true,
11154            });
11155            continue;
11156        }
11157        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
11158        let (code, body) = match spawned {
11159            Ok(mut running) => {
11160                let pid = running.child().id();
11161                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
11162                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
11163                let mut paths = vec![cwd.to_owned()];
11164                if let Some(cache) = state.config.cache_dir() {
11165                    paths.push(cache);
11166                }
11167                let mut pressure = None;
11168                let status = tokio::select! {
11169                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
11170                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
11171                        pressure = Some(reason); None
11172                    }
11173                };
11174                let (code, failure) = match status {
11175                    Some(Ok(Ok(status))) => (
11176                        status.code(),
11177                        status
11178                            .code()
11179                            .is_none()
11180                            .then(|| format!("command terminated: {status}")),
11181                    ),
11182                    Some(Ok(Err(error))) => {
11183                        (None, Some(format!("failed waiting for command: {error}")))
11184                    }
11185                    Some(Err(_)) => {
11186                        // Leased cleanup confirms process-tree death itself.
11187                        // Unleased commands still need the caller's PID check
11188                        // after kill-on-drop, before releasing cache ownership.
11189                        if !running.coordinated()
11190                            && let Some(pid) = pid
11191                        {
11192                            timed_out_pids.push(pid);
11193                        }
11194                        (
11195                            None,
11196                            Some(format!("timed out after {}s", timeout.as_secs())),
11197                        )
11198                    }
11199                    None => (None, None),
11200                };
11201                running.finish().await;
11202                if let Some(reason) = pressure {
11203                    state.resource_block = Some(crate::run::ResourceBlock {
11204                        cause: crate::run::ResourceBlockCause::Other,
11205                        node: node.to_owned(),
11206                        kind: "disk pressure".to_owned(),
11207                        detail: reason.clone(),
11208                        at: Timestamp::now(),
11209                    });
11210                    out.push(CommandOutcome {
11211                        command: command.clone(),
11212                        code: None,
11213                        output_tail: reason,
11214                        duration_ms: started.elapsed().as_millis() as u64,
11215                        resource_blocked: true,
11216                    });
11217                    break;
11218                }
11219                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11220                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11221                if let Some(failure) = failure {
11222                    body.push('\n');
11223                    body.push_str(&failure);
11224                }
11225                (code, body)
11226            }
11227            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11228        };
11229        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
11230        if !body.is_empty() {
11231            // The full text is kept whatever the summary dropped. A fresh
11232            // name per command so a resume or retry never overwrites an
11233            // earlier run's output. A failed write changes no verdict.
11234            match crate::run::write_verification_artifact(state, task, &body) {
11235                Ok(name) if summary != body => {
11236                    summary = format!("[full output: artifacts/{name}]\n{summary}");
11237                }
11238                Ok(_) => {}
11239                Err(e) => {
11240                    let warning = format!("could not save the full {task} output: {e:#}");
11241                    tracing::warn!("{warning}");
11242                    state.event(node, warning);
11243                    if summary != body {
11244                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
11245                    }
11246                }
11247            }
11248        }
11249        out.push(CommandOutcome {
11250            command: command.clone(),
11251            code,
11252            output_tail: summary,
11253            duration_ms: started.elapsed().as_millis() as u64,
11254            resource_blocked: false,
11255        });
11256    }
11257    state.task_finished(task);
11258    if let Err(e) = state.save() {
11259        tracing::warn!("could not persist the end of {task}: {e:#}");
11260    }
11261    (out, timed_out_pids)
11262}
11263
11264/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
11265/// section (`report::run`) and in the `merge` event this node records — for
11266/// the operator to run by hand.
11267///
11268/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
11269/// branch whose ruleset forbids merge commits (GitHub's "must not contain
11270/// merge commits", or "require linear history") rejects the push a `--no-ff`
11271/// merge would produce, which is exactly the guidance this function replaces.
11272/// `message`'s first line becomes the squash commit's subject, matching the
11273/// note `report::run` prints alongside this command — see that function for
11274/// why an explicit subject is not optional there.
11275fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11276    let repo = repo.display();
11277    match style {
11278        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11279        MergeStyle::Squash => {
11280            // The subject sits inside double quotes, and a title an agent
11281            // wrote may carry the characters that break out of them.
11282            let subject = message
11283                .lines()
11284                .next()
11285                .unwrap_or(branch)
11286                .replace(['\\', '"', '$', '`'], "");
11287            format!(
11288                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11289            )
11290        }
11291        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11292    }
11293}
11294
11295/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
11296/// under the hood, rejects a `title` over 256 characters and the whole
11297/// command fails — no PR at all, for a run whose body was otherwise fine
11298/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
11299/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
11300/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
11301/// this bound. It is a margin, not a guarantee — a title packed
11302/// with multi-unit characters could still in principle land close to the
11303/// edge, but a real task title's occasional emoji or accented letter fits
11304/// comfortably inside it.
11305const PR_TITLE_MAX: usize = 240;
11306
11307/// A pull request title from the opening line of `text`, or `None` when that
11308/// line is not English (GitHub text is) or has no letters.
11309///
11310/// A line within `max` is kept as is. A longer one is cut at the end of its
11311/// first sentence when that falls inside `max`, else at a word boundary with a
11312/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
11313/// make every merely-truncated title look non-English). The language check
11314/// runs on the kept text before any mark is added, so only what GitHub will
11315/// show is judged: an English opening followed by non-ASCII far past the cut
11316/// still passes.
11317fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11318    let line = queue::first_line(text)?;
11319    let chars: Vec<char> = line.chars().collect();
11320    let (kept, mark) = if chars.len() <= max {
11321        (line.to_owned(), "")
11322    } else if let Some(end) = sentence_end(&chars, max) {
11323        (chars[..end].iter().collect::<String>(), "")
11324    } else {
11325        let room = max.saturating_sub(3);
11326        // Cut at the last space inside the room; when the char just past the
11327        // room is a space the room already ends on a word.
11328        let cut = if chars[room].is_whitespace() {
11329            room
11330        } else {
11331            chars[..room]
11332                .iter()
11333                .rposition(|c| c.is_whitespace())
11334                .unwrap_or(room)
11335        };
11336        let head: String = chars[..cut].iter().collect();
11337        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11338        (head.to_owned(), "...")
11339    };
11340    if kept.is_empty() {
11341        return None;
11342    }
11343    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11344        return None;
11345    }
11346    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11347        return None;
11348    }
11349    Some(format!("{kept}{mark}"))
11350}
11351
11352/// The char length of the first sentence of `chars` when it ends within `max`
11353/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
11354/// abbreviations so `e.g. foo` does not end a title early.
11355fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11356    const MIN: usize = 20;
11357    for i in MIN..max.min(chars.len()) {
11358        if !matches!(chars[i], '.' | '!' | '?') {
11359            continue;
11360        }
11361        let Some(&next) = chars.get(i + 1) else {
11362            continue;
11363        };
11364        if !next.is_whitespace() {
11365            continue;
11366        }
11367        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11368        if after.is_some_and(|c| c.is_ascii_lowercase()) {
11369            continue;
11370        }
11371        let word: String = chars[..i]
11372            .iter()
11373            .rev()
11374            .take_while(|c| !c.is_whitespace())
11375            .collect::<Vec<_>>()
11376            .into_iter()
11377            .rev()
11378            .collect();
11379        let word = word.to_ascii_lowercase();
11380        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11381            continue;
11382        }
11383        let end = chars[..i]
11384            .iter()
11385            .rposition(|c| !c.is_whitespace())
11386            .map_or(i, |p| p + 1);
11387        return Some(end);
11388    }
11389    None
11390}
11391
11392/// What `merge = "pr"` (and the merge commit of the other modes) says about a
11393/// change: a title and a body describing what was *implemented*, not the task
11394/// that asked for it. A task reads as a request; a reader of the merged
11395/// history wants the change.
11396struct PrMessage {
11397    title: String,
11398    body: String,
11399}
11400
11401impl PrMessage {
11402    /// Title, blank line, body. The first line is the squash/merge commit
11403    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
11404    /// has to stay one sensible line.
11405    fn commit_message(&self) -> String {
11406        format!("{}\n\n{}", self.title, self.body)
11407    }
11408}
11409
11410/// The text after a leading `TITLE:` (any case) on `line`.
11411fn title_marker(line: &str) -> Option<&str> {
11412    let line = line.trim();
11413    let head = line.get(..6)?;
11414    head.eq_ignore_ascii_case("title:")
11415        .then(|| line[6..].trim())
11416}
11417
11418/// The implementer's own one-line title: the `TITLE:` line the implement
11419/// prompt asks for at the top of its SUMMARY. Candidate commits are all
11420/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
11421/// source, and a title that says as much is refused here too.
11422fn summary_title(summary: &str) -> Option<String> {
11423    let first = summary.lines().find(|l| !l.trim().is_empty())?;
11424    let raw = title_marker(first)?;
11425    if raw.is_empty() {
11426        return None;
11427    }
11428    let title = queue::title_from(raw, PR_TITLE_MAX);
11429    let lower = title.to_ascii_lowercase();
11430    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11431        return None;
11432    }
11433    Some(title)
11434}
11435
11436/// How `open_review`'s instruction begins; see [`landing_title`].
11437const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11438
11439/// Marker `open_review` gives a candidate that nothing in the roster wrote.
11440pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11441
11442/// Does this run review work that already existed, rather than implement a
11443/// task? Runs recorded before `reviewed_commits` existed carry only the
11444/// candidate marker.
11445pub(crate) fn is_review_run(state: &RunState) -> bool {
11446    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11447}
11448
11449/// The title of a review-only run: the subject of the oldest commit under
11450/// review. Later commits are usually fixups, and `instruction` is the review
11451/// prompt, which says nothing about the change. GitHub text is English, so a
11452/// non-ASCII or blank subject yields `None` and the caller's neutral title.
11453fn review_title(state: &RunState) -> Option<String> {
11454    english_subject(
11455        state.reviewed_commits.as_ref()?.first()?,
11456        state.config.graph.github_english(),
11457    )
11458}
11459
11460/// `raw` as a pull request title, or `None` when it is blank, not English
11461/// (GitHub text is), or one of magi's own candidate commit subjects.
11462fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11463    let raw = raw.trim();
11464    if raw.is_empty() {
11465        return None;
11466    }
11467    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11468        return None;
11469    }
11470    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11471        return None;
11472    }
11473    let title = queue::title_from(raw, PR_TITLE_MAX);
11474    let lower = title.to_ascii_lowercase();
11475    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11476        return None;
11477    }
11478    Some(title)
11479}
11480
11481/// What a review-only run's branch says about itself, read at the moment the
11482/// pull request is opened.
11483#[derive(Debug, Clone, PartialEq, Eq)]
11484struct BranchFacts {
11485    /// `(subject, body)` of each commit, oldest first.
11486    commits: Vec<(String, String)>,
11487    /// Trimmed `git diff --stat`.
11488    stat: String,
11489}
11490
11491/// Longest diff stat shown: this many file lines plus the summary line.
11492const STAT_FILE_LINES: usize = 25;
11493/// Cap on the commit list, well inside GitHub's 65536-character body limit.
11494const COMMITS_MAX_CHARS: usize = 20_000;
11495
11496/// Read the commits and diff stat of `base..branch`. `None` when git cannot
11497/// say or finds nothing, so the caller falls back to what the run recorded.
11498async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11499    let commits = git::commit_log(repo, base, branch).await.ok()?;
11500    if commits.is_empty() {
11501        return None;
11502    }
11503    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11504    let lines: Vec<&str> = stat.lines().collect();
11505    let stat = if lines.len() > STAT_FILE_LINES + 1 {
11506        let omitted = lines.len() - 1 - STAT_FILE_LINES;
11507        let more = format!(" ... {omitted} more file(s)");
11508        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11509        kept.push(&more);
11510        kept.push(lines[lines.len() - 1]);
11511        kept.join("\n")
11512    } else {
11513        lines.join("\n")
11514    };
11515    Some(BranchFacts { commits, stat })
11516}
11517
11518/// Defang what would break the surrounding markdown: a closing `</details>`
11519/// and a code fence.
11520fn markdown_safe(text: &str) -> String {
11521    text.replace("</details>", "&lt;/details&gt;")
11522        .replace("\x60\x60\x60", "~~~")
11523}
11524
11525fn neutral_title(state: &RunState, winner: char) -> String {
11526    format!(
11527        "chore: land candidate {} of run {}",
11528        winner.to_ascii_uppercase(),
11529        state.id
11530    )
11531}
11532
11533/// Reuse the graph's bounded fixer without starting another graph or driver.
11534/// Copy the working state back even on errors so spent rounds remain visible.
11535pub(crate) async fn repair_landing_gate(
11536    state: &mut RunState,
11537    winner: &Candidate,
11538    outcomes: &[CommandOutcome],
11539) -> Result<GateFix> {
11540    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11541        return Ok(GateFix::Stop);
11542    }
11543    let mut runner = Runner {
11544        roles: state.config.resolve_roles()?,
11545        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11546        state: state.clone(),
11547        pause: Pause::new(),
11548        interrupt: Pause::new(),
11549    };
11550    let result = runner.gate_fix_round(winner, outcomes).await;
11551    *state = runner.state;
11552    result
11553}
11554
11555/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
11556/// A completed failure is terminal on that SHA. Resource waits remain retryable.
11557pub(crate) async fn verify_current_head(
11558    state: &mut RunState,
11559    winner: &Candidate,
11560    wait: &crate::repo_lock::WaitPolicy,
11561) -> Result<bool> {
11562    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11563    // Preserve the previous record before replacing it, including on crash recovery.
11564    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
11565        state
11566            .verification_history
11567            .push(crate::run::VerificationEvidence {
11568                superseded_by: Some(head.clone()),
11569                archived_at: Some(Timestamp::now()),
11570                fixes_spent: Some(state.gate_fixes.len()),
11571                gate_head: None,
11572                gate_ran: false,
11573                gate: Vec::new(),
11574                e2e: state.head_e2e.take(),
11575            });
11576    }
11577    if let Some(e) = &state.head_e2e
11578        && e.head == head
11579    {
11580        if e.passed {
11581            return Ok(true);
11582        }
11583        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
11584            state.status = RunStatus::Blocked;
11585            state.save()?;
11586            return Ok(false);
11587        }
11588    } else if state.reviews.last().is_some_and(|r| {
11589        r.verified_head.as_deref() == Some(head.as_str())
11590            && (r.e2e_status() == E2eStatus::Passed
11591                || (r.e2e_status() == E2eStatus::NotConfigured
11592                    && state.config.verify.e2e.is_empty()))
11593    }) || state
11594        .gate_repair_e2e
11595        .as_ref()
11596        .is_some_and(|e| e.head == head && e.passed)
11597    {
11598        return Ok(true);
11599    }
11600    if let Some(e) = &state.gate_repair_e2e
11601        && e.head == head
11602        && !e.passed
11603        && !e.outcomes.is_empty()
11604        && !verify_inconclusive(&e.outcomes)
11605    {
11606        state.status = RunStatus::Blocked;
11607        state.save()?;
11608        return Ok(false);
11609    }
11610    state.head_e2e = Some(crate::run::GateRepairE2e {
11611        head: head.clone(),
11612        passed: false,
11613        outcomes: Vec::new(),
11614    });
11615    state.save()?;
11616    let commands = state.config.verify.e2e.clone();
11617    let shell = state.config.shell();
11618    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11619    let cache_dir = state.config.cache_dir();
11620    let outcomes = if commands.is_empty() {
11621        Vec::new()
11622    } else {
11623        let (outcomes, _) = with_cache_lease(
11624            state,
11625            cache_dir.as_deref(),
11626            "e2e",
11627            "e2e",
11628            &winner.worktree,
11629            &head,
11630            timeout,
11631            "current head e2e",
11632            wait,
11633            |state, budget| async move {
11634                run_e2e_with_retry(
11635                    state,
11636                    &shell,
11637                    &commands,
11638                    &winner.worktree,
11639                    budget,
11640                    "current head e2e",
11641                )
11642                .await
11643            },
11644        )
11645        .await?;
11646        outcomes
11647    };
11648    let unresolved = verify_inconclusive(&outcomes);
11649    let passed = !unresolved
11650        && outcomes.iter().all(CommandOutcome::ok)
11651        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11652    state.head_e2e = Some(crate::run::GateRepairE2e {
11653        head: head.clone(),
11654        passed,
11655        outcomes,
11656    });
11657    if !passed && !unresolved {
11658        state.status = RunStatus::Blocked;
11659        state.event(
11660            "e2e",
11661            format!("verify.e2e failed on head {head}; nothing published"),
11662        );
11663    }
11664    state.save()?;
11665    Ok(passed)
11666}
11667
11668/// Recheck a repair's e2e requirement across failures, resource deferrals and
11669/// process interruptions. A successful result only applies to its exact SHA.
11670pub(crate) async fn check_gate_repair_e2e(
11671    state: &mut RunState,
11672    winner: &Candidate,
11673    wait: &crate::repo_lock::WaitPolicy,
11674) -> Result<GateFix> {
11675    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11676        return Ok(GateFix::Retry);
11677    }
11678    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11679    if state
11680        .gate_repair_e2e
11681        .as_ref()
11682        .is_some_and(|r| r.head == head && r.passed)
11683    {
11684        return Ok(GateFix::Retry);
11685    }
11686    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11687        head: head.clone(),
11688        passed: false,
11689        outcomes: Vec::new(),
11690    });
11691    state.save()?;
11692    let commands = state.config.verify.e2e.clone();
11693    let outcomes = if commands.is_empty() {
11694        Vec::new()
11695    } else {
11696        let shell = state.config.shell();
11697        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11698        let cache_dir = state.config.cache_dir();
11699        let (outcomes, _) = with_cache_lease(
11700            state,
11701            cache_dir.as_deref(),
11702            "e2e",
11703            "e2e",
11704            &winner.worktree,
11705            &head,
11706            timeout,
11707            "gate repair e2e",
11708            wait,
11709            |state, budget| async move {
11710                run_e2e_with_retry(
11711                    state,
11712                    &shell,
11713                    &commands,
11714                    &winner.worktree,
11715                    budget,
11716                    "gate repair e2e",
11717                )
11718                .await
11719            },
11720        )
11721        .await?;
11722        outcomes
11723    };
11724    let deferred = verify_inconclusive(&outcomes);
11725    let passed = outcomes.iter().all(CommandOutcome::ok)
11726        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11727    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11728        head,
11729        passed,
11730        outcomes,
11731    });
11732    state.save()?;
11733    if deferred {
11734        Ok(GateFix::Defer)
11735    } else if !passed {
11736        state.event(
11737            "gate",
11738            "verify.e2e failed after gate repair; nothing published",
11739        );
11740        Ok(GateFix::Stop)
11741    } else {
11742        Ok(GateFix::Retry)
11743    }
11744}
11745
11746/// Run the same configured gate for a landing head and a fresh graph.
11747pub(crate) async fn run_landing_gate(
11748    state: &mut RunState,
11749    winner: &Candidate,
11750    wait: &crate::repo_lock::WaitPolicy,
11751) -> Result<Vec<CommandOutcome>> {
11752    let shell = state.config.shell();
11753    let gate_commands = state.config.verify.gate.clone();
11754    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11755    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11756        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
11757        Vec::new()
11758    } else {
11759        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11760        let cache_dir = state.config.cache_dir();
11761        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11762        let (outcomes, _) = with_cache_lease(
11763            state,
11764            cache_dir.as_deref(),
11765            "gate",
11766            "gate",
11767            &winner.worktree,
11768            &head,
11769            timeout,
11770            "final gate",
11771            wait,
11772            |state, budget| {
11773                let shell = shell.clone();
11774                let gate_commands = gate_commands.clone();
11775                let winner = winner.clone();
11776                async move {
11777                    Runner::run_pre_gate(state, &winner).await;
11778                    // Bind command evidence after mechanical fixes, before commands start.
11779                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
11780                    let (outcomes, timed_out_pids) = run_commands(
11781                        state,
11782                        "gate",
11783                        "gate",
11784                        0,
11785                        &shell,
11786                        &gate_commands,
11787                        &winner.worktree,
11788                        budget,
11789                    )
11790                    .await;
11791                    (outcomes, false, timed_out_pids)
11792                }
11793            },
11794        )
11795        .await?;
11796        outcomes
11797    };
11798    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
11799    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
11800        let mut outcomes = outcomes;
11801        outcomes.push(CommandOutcome {
11802            command: "gate head stability".into(), code: None,
11803            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
11804            duration_ms: 0, resource_blocked: false,
11805        });
11806        return Ok(outcomes);
11807    }
11808    if outcomes.is_empty() {
11809        // Nothing configured to check — distinct from every other
11810        // silence in this run's event log, since an empty `gate` alone
11811        // no longer says whether the gate ran at all (see
11812        // `RunState::gate_ran`'s own doc).
11813        state.event("gate", "no gate commands configured; not tested");
11814    }
11815    for o in &outcomes {
11816        state.event(
11817            "gate",
11818            format!(
11819                "`{}` -> {}",
11820                o.command,
11821                if o.ok() {
11822                    "pass".to_owned()
11823                } else {
11824                    format!(
11825                        "FAIL ({:?})\n{}",
11826                        o.code,
11827                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11828                    )
11829                }
11830            ),
11831        );
11832    }
11833    Ok(outcomes)
11834}
11835
11836/// The pull request title to hand to `land::merge_subject`. A review-only run
11837/// opened by an earlier build titled its pull request with the review prompt;
11838/// that title is dropped (empty, so the fallback applies) rather than landed.
11839/// Any other title, including an operator's rename, passes through untouched,
11840/// and so does every title of a run that implements a task.
11841pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11842    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11843        ""
11844    } else {
11845        pr_title
11846    }
11847}
11848
11849/// What the squash subject falls back to when the pull request title is empty
11850/// or candidate-shaped: for a review-only run the derived title, never the
11851/// review prompt held in `instruction`.
11852pub fn landing_subject_source(state: &RunState) -> String {
11853    if is_review_run(state) {
11854        let winner = state.candidates.first().map_or('A', |c| c.label);
11855        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11856    }
11857    state.instruction.clone()
11858}
11859
11860/// `summary` without its `TITLE:` line, which the pull request title already
11861/// carries.
11862fn summary_without_title(summary: &str) -> String {
11863    let mut lines = summary.trim().lines().peekable();
11864    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11865        lines.next();
11866    }
11867    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11868}
11869
11870/// The pull request title and body for the winning candidate.
11871///
11872/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
11873/// the task's own opening line via [`queue::title_from`] when there is none.
11874/// `state.instruction` can open with blank lines (`task_text` only rejects a
11875/// body that is blank *entirely*), which `title_from` skips.
11876///
11877/// Publishing uses an independent description generated from the final diff
11878/// and structured evidence. Internal summaries and reports remain local.
11879/// Non-publishing previews may display the original task and candidate summary.
11880/// The footer retains the run and candidate provenance.
11881#[cfg(test)]
11882fn pr_message(state: &RunState, winner: char) -> PrMessage {
11883    pr_message_with(state, winner, None)
11884}
11885
11886/// [`pr_message`] with what the branch of a review-only run says about itself.
11887/// `facts` is ignored for a run that implements a task.
11888#[cfg(test)]
11889fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11890    let raw = pr_message_raw(
11891        state,
11892        winner,
11893        facts,
11894        crate::pr_outcome::FinalHead::Unreported,
11895        ChangeNote::NotAsked,
11896    );
11897    let id = crate::scrub::Identity::current();
11898    PrMessage {
11899        title: crate::scrub::scrub(&raw.title, &id),
11900        body: crate::scrub::scrub(&raw.body, &id),
11901    }
11902}
11903
11904/// What the public description may say about the change itself.
11905#[derive(Debug, Clone, Copy)]
11906enum ChangeNote<'a> {
11907    /// A local preview or non-publishing merge mode: no public generation needed.
11908    NotAsked,
11909    /// A summary written from the final diff.
11910    Summary(&'a crate::run::ChangeSummary),
11911    /// A summary was asked for and could not be produced; the text says why.
11912    Unavailable(&'a str),
11913}
11914
11915fn pr_message_raw(
11916    state: &RunState,
11917    winner: char,
11918    facts: Option<&BranchFacts>,
11919    head: crate::pr_outcome::FinalHead<'_>,
11920    change: ChangeNote<'_>,
11921) -> PrMessage {
11922    if !matches!(change, ChangeNote::NotAsked) {
11923        let (title, mut body) = match change {
11924            ChangeNote::Summary(c) => (
11925                c.title.clone(),
11926                format!("## Summary\n\n{}\n\n", c.body.trim()),
11927            ),
11928            ChangeNote::Unavailable(why) => (
11929                neutral_title(state, winner),
11930                format!(
11931                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11932                ),
11933            ),
11934            ChangeNote::NotAsked => unreachable!(),
11935        };
11936        body.push_str(&public_pr_evidence(state, head));
11937        body.push_str(&format!(
11938            "\n---\nmagi:run/{} magi:candidate-{}\n",
11939            state.id,
11940            winner.to_ascii_lowercase()
11941        ));
11942        return PrMessage { title, body };
11943    }
11944    pr_preview(state, winner, facts, head)
11945}
11946
11947/// Append public facts from the run records after every prose generation,
11948/// including posting-gate rewrites. The agent cannot drop this provenance.
11949fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11950    let mut body = crate::pr_outcome::review_rounds(state);
11951    body.push_str(&crate::pr_outcome::final_verification(state, head));
11952    let open = state.last_round_findings();
11953    if !open.is_empty() {
11954        body.push_str("## Open review findings\n\n");
11955        for f in open {
11956            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11957        }
11958        body.push('\n');
11959    }
11960    if let Some(c) = state.change_summary.as_ref()
11961        && head == crate::pr_outcome::FinalHead::At(&c.head)
11962    {
11963        body.push_str(&format!(
11964            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11965            c.base, c.head
11966        ));
11967    }
11968    body
11969}
11970
11971/// Local-only preview of the operator's task and candidate summary.
11972fn pr_preview(
11973    state: &RunState,
11974    winner: char,
11975    facts: Option<&BranchFacts>,
11976    head: crate::pr_outcome::FinalHead<'_>,
11977) -> PrMessage {
11978    let summary = state
11979        .candidates
11980        .iter()
11981        .find(|c| c.label == winner)
11982        .map(|c| c.summary.as_str())
11983        .unwrap_or_default();
11984    // The fallback is the operator's own words and may not be English; GitHub
11985    // text always is, so a non-English task gets a neutral title instead.
11986    let review = is_review_run(state);
11987    let title = if review {
11988        facts
11989            .and_then(|f| {
11990                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11991            })
11992            .or_else(|| review_title(state))
11993            .or_else(|| {
11994                state
11995                    .candidates
11996                    .iter()
11997                    .find(|c| c.label == winner)
11998                    .filter(|c| !c.branch.starts_with("magi/"))
11999                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12000            })
12001            .unwrap_or_else(|| neutral_title(state, winner))
12002    } else {
12003        summary_title(summary).unwrap_or_else(|| {
12004            english_title(
12005                &state.instruction,
12006                PR_TITLE_MAX,
12007                state.config.graph.github_english(),
12008            )
12009            .unwrap_or_else(|| neutral_title(state, winner))
12010        })
12011    };
12012
12013    let mut body = String::new();
12014    let what = summary_without_title(summary);
12015    if !what.is_empty() {
12016        body.push_str("## Summary\n\n");
12017        body.push_str(&what);
12018        body.push_str("\n\n");
12019    }
12020
12021    // Review history uses structured facts across all rounds, never reports.
12022    body.push_str(&crate::pr_outcome::review_rounds(state));
12023    body.push_str(&crate::pr_outcome::final_verification(state, head));
12024    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12025
12026    let open = state.open_findings();
12027    if !open.is_empty() {
12028        body.push_str("## Open review findings\n\n");
12029        for f in &open {
12030            body.push_str(&format!(
12031                "- `{}` [{:?}] {}\n",
12032                f.id, f.severity, "Context retained in local review records."
12033            ));
12034        }
12035        body.push('\n');
12036    }
12037
12038    if let Some(fix) = fix
12039        && !fix.rejected.is_empty()
12040    {
12041        body.push_str("## Declined by the fixer\n\n");
12042        for r in &fix.rejected {
12043            body.push_str(&format!(
12044                "- `{}`: {}\n",
12045                r.id, "Reason retained in local review records."
12046            ));
12047        }
12048        body.push('\n');
12049    }
12050
12051    if review {
12052        // The review prompt is not the task; list what the branch carries.
12053        body.push_str("## Commits under review\n\n");
12054        if let Some(facts) = facts {
12055            let mut left = COMMITS_MAX_CHARS;
12056            for (i, (subject, text)) in facts.commits.iter().enumerate() {
12057                let mut entry = format!("- {}\n", markdown_safe(subject));
12058                for l in markdown_safe(text).lines() {
12059                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
12060                }
12061                if left == 0 {
12062                    body.push_str(&format!(
12063                        "- ... {} more commit(s)\n",
12064                        facts.commits.len() - i
12065                    ));
12066                    break;
12067                }
12068                if entry.len() > left {
12069                    // Even the first commit is cut: one huge body must not
12070                    // push the whole description past GitHub's limit.
12071                    let mut end = left;
12072                    while !entry.is_char_boundary(end) {
12073                        end -= 1;
12074                    }
12075                    entry.truncate(end);
12076                    entry.push_str("\n  ... (truncated)\n");
12077                    left = 0;
12078                } else {
12079                    left -= entry.len();
12080                }
12081                body.push_str(&entry);
12082            }
12083            if !facts.stat.trim().is_empty() {
12084                body.push_str(&format!(
12085                    "\n## Diff stat\n\n```\n{}\n```\n",
12086                    markdown_safe(facts.stat.trim())
12087                ));
12088            }
12089        } else {
12090            match &state.reviewed_commits {
12091                Some(subjects) => {
12092                    for s in subjects {
12093                        body.push_str(&format!("- {}\n", s.trim()));
12094                    }
12095                }
12096                None => {
12097                    // An older run kept only the prompt, with the commit list
12098                    // after its first paragraph.
12099                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
12100                    body.push_str(rest.trim());
12101                    body.push('\n');
12102                }
12103            }
12104        }
12105    } else {
12106        let task = state.instruction.trim();
12107        let task = if task.is_empty() {
12108            "(empty task)"
12109        } else {
12110            task
12111        };
12112        body.push_str(&format!(
12113            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
12114            task.replace("</details>", "&lt;/details&gt;")
12115        ));
12116    }
12117
12118    body.push_str(&format!(
12119        "\n---\nmagi:run/{} magi:candidate-{}\n",
12120        state.id,
12121        winner.to_ascii_lowercase()
12122    ));
12123
12124    PrMessage { title, body }
12125}
12126
12127/// Source imports have already been prepared from structured inputs. Legacy
12128/// inferred references must never instruct a seat to reconstruct those imports.
12129fn seeded_instruction(state: &RunState) -> String {
12130    let mut inputs = Vec::new();
12131    if let Some(start) = &state.sources.start_from {
12132        inputs.push(format!(
12133            "Start from `{}` at `{}` (including ancestry).",
12134            start.token, start.sha
12135        ));
12136    }
12137    for pick in &state.sources.pick_commits {
12138        inputs.push(format!(
12139            "Pick `{}` at `{}` (single commit).",
12140            pick.token, pick.sha
12141        ));
12142    }
12143    if inputs.is_empty() {
12144        state.instruction.clone()
12145    } else {
12146        format!(
12147            "{}\n\n# Explicit source inputs already prepared\n\n{}",
12148            state.instruction,
12149            inputs.join("\n")
12150        )
12151    }
12152}
12153
12154/// Does the winner have no commits ahead of the base it would land on?
12155/// Any failure to find out reads as "not empty": the merge then behaves as it
12156/// always did rather than refusing on a guess.
12157async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
12158    let base = &state.base_branch;
12159    let mut against = base.clone();
12160    if mode == MergeMode::Pr {
12161        // A pull request lands on the remote's base, never the local branch
12162        // of the same name: if that cannot be read, "not empty" is the safe
12163        // answer.
12164        let remote = &state.config.merge.remote;
12165        let tracking = format!("{remote}/{base}");
12166        let fetched = git::fetch(repo, remote, base).await;
12167        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
12168            against = tracking;
12169        } else {
12170            return false;
12171        }
12172    }
12173    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
12174}
12175
12176/// Why nothing was opened for an empty winner, with what the task's own
12177/// references resolved to.
12178fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
12179    let mut detail = format!(
12180        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
12181         nothing to open a pull request for"
12182    );
12183    detail.push_str(
12184        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
12185    );
12186    detail
12187}
12188
12189/// What the `Pr` merge does once it knows whether the branch already has an
12190/// open pull request.
12191#[derive(Debug, PartialEq, Eq)]
12192enum PrPlan {
12193    Create,
12194    Adopt { url: String, title: String },
12195    Stop(String),
12196}
12197
12198/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
12199/// a failed lookup -> stop with the real reason. Never guesses.
12200fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
12201    match found {
12202        Ok(land::OpenPr::None) => PrPlan::Create,
12203        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
12204        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
12205            "several open pull requests exist for this branch, not picking one: {}",
12206            urls.join(" ")
12207        )),
12208        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
12209    }
12210}
12211
12212/// `gh pr create`, returning the PR url.
12213pub(crate) async fn gh_pr_create(
12214    cwd: &Path,
12215    base: &str,
12216    head: &str,
12217    title: &str,
12218    body: &str,
12219) -> Result<String> {
12220    let out = tokio::process::Command::new("gh")
12221        .args([
12222            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
12223        ])
12224        .current_dir(cwd)
12225        .quiet()
12226        .stdin(std::process::Stdio::null())
12227        .output()
12228        .await
12229        .context("spawn gh")?;
12230    if out.status.success() {
12231        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
12232    } else {
12233        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
12234    }
12235}
12236
12237/// Never force deletion of uncommitted work, including unknown legacy trees.
12238async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
12239    match std::fs::symlink_metadata(path) {
12240        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
12241            // A previous cleanup may have removed the tree before a branch
12242            // deletion or state save failed. Unregister it if still present.
12243            git::worktree_remove_clean(repo, path).await?;
12244            return Ok(());
12245        }
12246        Err(e) => return Err(e.into()),
12247        Ok(_) => {}
12248    }
12249    if !git::is_clean(path).await? {
12250        bail!(
12251            "worktree has uncommitted work; refusing removal: {}",
12252            path.display()
12253        );
12254    }
12255    if !git::worktree_remove_clean(repo, path).await? {
12256        bail!("worktree removal refused: {}", path.display());
12257    }
12258    Ok(())
12259}
12260
12261/// Tear a run's worktrees and branches down.
12262///
12263/// `home` is where the updated `run.json` is saved (via
12264/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
12265/// a housekeeping pass already has its own honest `home` handed to it, and
12266/// falling through to the global here would write back through whichever
12267/// directory some other process or test pinned into that `OnceLock` first,
12268/// not the one the caller actually resolved its `runs` and `state` from.
12269pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
12270    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
12271        bail!("refusing fold: {reason}");
12272    }
12273    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
12274        bail!("candidate preservation incomplete; refusing cleanup");
12275    }
12276    // Serialize ordinary folding with the same starts/resumes as target-only
12277    // eviction. Otherwise a second janitor could remove the whole worktree
12278    // around a live output borrower even though target eviction was fenced.
12279    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
12280    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
12281    let _driver = crate::driver::maintenance(home, &state.id)?;
12282    let mut output_guards = Vec::new();
12283    for entry in std::fs::read_dir(state.worktree_root())
12284        .into_iter()
12285        .flatten()
12286        .flatten()
12287    {
12288        if entry.path().join(".git").is_file() {
12289            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
12290        }
12291    }
12292    let repo = state.repo.clone();
12293    // CLI, Web and automatic cleanup all enter here, independently of
12294    // Runner::fold_losers. Protect deletion and shared config release too.
12295    let wait = repository_wait(state, None);
12296    let _coordination = match coordinate_operation(state, &repo, &wait).await {
12297        Ok(lease) => lease,
12298        Err(e) => {
12299            state.save_under(home)?;
12300            return Err(e);
12301        }
12302    };
12303    // Confirmed merges dispose of their owned sources after durable evidence
12304    // capture. Preflight every source before removing anything.
12305    let _source_outputs = if state.status == RunStatus::Merged
12306        && state.merge.as_ref().is_some_and(|merge| merge.ok)
12307    {
12308        verify_merged_fold_sources(state, drop_winner, home).await?
12309    } else {
12310        Vec::new()
12311    };
12312    let root = state.worktree_root();
12313    let winner = state.tally.as_ref().map(|t| t.winner);
12314    let mut removed = Vec::new();
12315
12316    for i in 0..state.candidates.len() {
12317        let c = state.candidates[i].clone();
12318        let is_winner = Some(c.label) == winner;
12319        if is_winner && !drop_winner {
12320            continue;
12321        }
12322        let existed = c.worktree.exists();
12323        // A vanished directory can still have a Git registration that keeps
12324        // its branch checked out. Remove that registration before the branch.
12325        remove_preserved_worktree(&repo, &c.worktree).await?;
12326        if existed {
12327            removed.push(c.worktree.to_string_lossy().into_owned());
12328        }
12329        // A branch handed to a later run (and its pull request) is not this
12330        // run's to delete.
12331        let handed_over = state.released_branches.contains(&c.branch);
12332        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12333            if !git::branch_delete(&repo, &c.branch).await? {
12334                bail!("candidate branch removal refused");
12335            }
12336            removed.push(c.branch.clone());
12337        }
12338        state.candidates[i].folded = true;
12339    }
12340
12341    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12342        let path = name.path();
12343        let keep = !drop_winner
12344            && winner.is_some_and(|w| {
12345                path.file_name()
12346                    .is_some_and(|n| n == format!("cand-{w}").as_str())
12347            });
12348        if keep {
12349            continue;
12350        }
12351        remove_preserved_worktree(&repo, &path).await?;
12352        removed.push(path.to_string_lossy().into_owned());
12353    }
12354
12355    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
12356    // judge worktrees, so once the loop above has cleared all of them out,
12357    // the parent is a bare directory nobody else was ever going to remove -
12358    // git only ever managed what was inside it. Left alone, one of these
12359    // accumulates per fully-folded run; the operator's own machine had 74.
12360    // `remove_if_empty` re-checks rather than assuming: a run whose winner
12361    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
12362    // so does anything a run never claimed that happens to share the bay.
12363    remove_if_empty(&root);
12364
12365    if state.enabled_worktree_config && drop_winner {
12366        // A release, not a raw disable: some sibling run in this repository
12367        // may still hold its own reference (see `git::acquire_worktree_config`),
12368        // and only the last release actually turns the setting back off.
12369        git::release_worktree_config(&repo).await.ok();
12370        state.enabled_worktree_config = false;
12371    }
12372    state.save_under(home)?;
12373    Ok(removed)
12374}
12375
12376async fn verify_merged_fold_sources(
12377    state: &RunState,
12378    drop_winner: bool,
12379    home: &Path,
12380) -> Result<Vec<std::fs::File>> {
12381    let winner = state.tally.as_ref().map(|t| t.winner);
12382    let mut heads = BTreeSet::new();
12383    for candidate in &state.candidates {
12384        if Some(candidate.label) == winner && !drop_winner {
12385            continue;
12386        }
12387        anyhow::ensure!(
12388            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12389            "candidate recovery source is outside the inventoried run root: {}",
12390            candidate.worktree.display()
12391        );
12392        match std::fs::symlink_metadata(&candidate.worktree) {
12393            Ok(_) => {}
12394            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12395            Err(error) => return Err(error.into()),
12396        }
12397        if !state.released_branches.contains(&candidate.branch)
12398            && git::branch_exists(&state.repo, &candidate.branch).await?
12399        {
12400            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12401        }
12402    }
12403    let mut cargo_guards = Vec::new();
12404    let entries = match std::fs::read_dir(state.worktree_root()) {
12405        Ok(entries) => Some(entries),
12406        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12407        Err(error) => return Err(error.into()),
12408    };
12409    for entry in entries.into_iter().flatten() {
12410        let path = entry?.path();
12411        let metadata = std::fs::symlink_metadata(&path)?;
12412        anyhow::ensure!(
12413            metadata.is_dir() && !metadata.file_type().is_symlink(),
12414            "non-directory or symlink recovery source retained: {}",
12415            path.display()
12416        );
12417        if !drop_winner
12418            && winner.is_some_and(|w| {
12419                path.file_name()
12420                    .is_some_and(|name| name == format!("cand-{w}").as_str())
12421            })
12422        {
12423            continue;
12424        }
12425        anyhow::ensure!(
12426            path.join(".git").is_file(),
12427            "unregistered recovery source retained: {}",
12428            path.display()
12429        );
12430        anyhow::ensure!(
12431            git::git(
12432                &path,
12433                &[
12434                    "status",
12435                    "--porcelain",
12436                    "--untracked-files=all",
12437                    "--ignored",
12438                    "--",
12439                    ".",
12440                    ":(exclude)target"
12441                ]
12442            )
12443            .await?
12444            .is_empty(),
12445            "source changes or ignored recovery files retained: {}",
12446            path.display()
12447        );
12448        match std::fs::symlink_metadata(path.join("target")) {
12449            Ok(_) => {
12450                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
12451            }
12452            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12453            Err(error) => return Err(error.into()),
12454        }
12455        for checkpoint in [
12456            "rebase-merge",
12457            "rebase-apply",
12458            "sequencer",
12459            "CHERRY_PICK_HEAD",
12460            "MERGE_HEAD",
12461            "REBASE_HEAD",
12462            "REVERT_HEAD",
12463            "BISECT_LOG",
12464            "AUTO_MERGE",
12465            "MERGE_MSG",
12466        ] {
12467            let location = git::git(
12468                &path,
12469                &[
12470                    "rev-parse",
12471                    "--path-format=absolute",
12472                    "--git-path",
12473                    checkpoint,
12474                ],
12475            )
12476            .await?;
12477            anyhow::ensure!(
12478                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
12479                if e.kind() == std::io::ErrorKind::NotFound),
12480                "Git recovery checkpoint retained: {checkpoint}"
12481            );
12482        }
12483        heads.insert(git::rev_parse(&path, "HEAD").await?);
12484    }
12485    crate::fold_evidence::preserve(state, home, &heads).await?;
12486
12487    Ok(cargo_guards)
12488}
12489
12490/// Remove `dir` if it exists and has nothing in it.
12491///
12492/// Best-effort and silent by design: a directory that is not empty (a run
12493/// whose winner is still parked there, a stray file some other process left)
12494/// is exactly the case this must refuse, and a directory that is already gone
12495/// is not a failure worth reporting either. `std::fs::remove_dir` itself
12496/// already refuses a non-empty directory, so the emptiness check below is
12497/// belt, not suspenders - it is what keeps this from ever attempting the
12498/// removal in the case that matters, rather than trusting `remove_dir`'s
12499/// error path to have no side effects if it ever changed.
12500fn remove_if_empty(dir: &Path) {
12501    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
12502        std::fs::remove_dir(dir).ok();
12503    }
12504}
12505
12506/// Severity of the worst open finding in the last review round, for reporting.
12507pub fn worst_open(state: &RunState) -> Option<Severity> {
12508    state
12509        .reviews
12510        .last()?
12511        .reviews
12512        .iter()
12513        .flat_map(|r| r.findings.iter())
12514        .map(|f| f.severity)
12515        .max()
12516}
12517
12518/// `[worktree] setup` hides its products per worktree, which needs
12519/// `extensions.worktreeConfig`. Taken once per run and released with the run's
12520/// worktrees, like the hook's; every place that runs setup calls this first.
12521async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12522    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12523        git::acquire_worktree_config(repo).await?;
12524        state.enabled_worktree_config = true;
12525    }
12526    Ok(())
12527}
12528
12529#[cfg(test)]
12530mod tests {
12531    #[tokio::test]
12532    async fn launcher_base_fetch_requires_remote_success() {
12533        let tmp = tempfile::tempdir().unwrap();
12534        let repo = tmp.path().join("repo");
12535        let remote = tmp.path().join("remote");
12536        std::fs::create_dir(&repo).unwrap();
12537        std::fs::create_dir(&remote).unwrap();
12538        git::git(&remote, &["init", "--bare", "-b", "main"])
12539            .await
12540            .unwrap();
12541        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12542        git::git(
12543            &repo,
12544            &[
12545                "-c",
12546                "user.name=Fixture",
12547                "-c",
12548                "user.email=fixture@example.invalid",
12549                "commit",
12550                "--allow-empty",
12551                "-m",
12552                "fixture",
12553            ],
12554        )
12555        .await
12556        .unwrap();
12557        git::git(
12558            &repo,
12559            &["remote", "add", "origin", remote.to_str().unwrap()],
12560        )
12561        .await
12562        .unwrap();
12563        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12564        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12565        assert_eq!(
12566            resolve_base(&repo, "main", "origin", None)
12567                .await
12568                .unwrap()
12569                .commit,
12570            head
12571        );
12572        #[cfg(unix)]
12573        {
12574            let policy: crate::config::Coordination = toml::from_str(
12575                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12576            ).unwrap();
12577            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12578            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12579            let held = crate::repo_lock::SeatLease::operation_with(
12580                &repo,
12581                Some(&policy),
12582                &crate::repo_lock::WaitPolicy::default(),
12583            )
12584            .await
12585            .unwrap();
12586            assert!(held.is_some());
12587            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12588            tokio::pin!(fetch);
12589            assert!(
12590                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12591                    .await
12592                    .is_err()
12593            );
12594            assert!(
12595                !repo.join(".git/FETCH_HEAD").exists(),
12596                "no fetch before admission"
12597            );
12598            drop(held);
12599            let fetched = fetch.await.unwrap();
12600            assert_eq!(fetched.commit, head);
12601            assert_eq!(fetched.remote, "origin");
12602            assert_eq!(fetched.branch, "main");
12603        }
12604        // A surviving tracking ref is not successful fetch evidence.
12605        git::git(
12606            &repo,
12607            &[
12608                "remote",
12609                "set-url",
12610                "origin",
12611                tmp.path().join("missing").to_str().unwrap(),
12612            ],
12613        )
12614        .await
12615        .unwrap();
12616        let error = resolve_base(&repo, "main", "origin", None)
12617            .await
12618            .unwrap_err()
12619            .to_string();
12620        assert!(error.contains("cannot read origin/main"));
12621        assert!(error.contains("refusing to branch off"));
12622    }
12623
12624    #[test]
12625    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12626        let leaked = vec!["chore(deps): update a crate".to_owned()];
12627        let own = "fix(daemon): apply a chosen action";
12628        assert!(should_retitle("", own, &leaked));
12629        assert!(should_retitle(
12630            &format!("{REVIEW_PROMPT_OPENING} `x`"),
12631            own,
12632            &leaked
12633        ));
12634        assert!(should_retitle(
12635            "chore: land candidate A of run 1",
12636            own,
12637            &leaked
12638        ));
12639        assert!(should_retitle(
12640            "magi: candidate A (uncommitted work)",
12641            own,
12642            &leaked
12643        ));
12644        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12645        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12646        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12647    }
12648
12649    #[test]
12650    fn pr_merge_plan_creates_adopts_or_stops() {
12651        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12652        assert_eq!(
12653            pr_merge_plan(Ok(land::OpenPr::One {
12654                url: "u".into(),
12655                title: "t".into()
12656            })),
12657            PrPlan::Adopt {
12658                url: "u".into(),
12659                title: "t".into()
12660            }
12661        );
12662        let PrPlan::Stop(many) =
12663            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12664        else {
12665            panic!("many must stop");
12666        };
12667        assert!(many.contains('a') && many.contains('b'));
12668        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12669            panic!("a failed lookup must stop");
12670        };
12671        assert!(err.contains("bad token"));
12672    }
12673
12674    use super::*;
12675    use crate::run::GateStatus;
12676    use std::collections::BTreeMap;
12677    use std::time::Duration;
12678
12679    fn conductor() -> AgentSpec {
12680        AgentSpec {
12681            id: "conductor".to_owned(),
12682            kind: crate::config::AgentKind::Command,
12683            model: None,
12684            command: vec!["true".to_owned()],
12685            extra_args: Vec::new(),
12686            env: BTreeMap::new(),
12687            prompt_delivery: None,
12688            max_turn_secs: None,
12689        }
12690    }
12691
12692    fn spec(id: &str) -> AgentSpec {
12693        AgentSpec {
12694            id: id.to_owned(),
12695            kind: crate::config::AgentKind::Command,
12696            model: None,
12697            command: vec!["true".to_owned()],
12698            extra_args: Vec::new(),
12699            env: BTreeMap::new(),
12700            prompt_delivery: None,
12701            max_turn_secs: None,
12702        }
12703    }
12704
12705    fn ids(xs: &[&str]) -> BTreeSet<String> {
12706        xs.iter().map(|s| (*s).to_owned()).collect()
12707    }
12708
12709    #[test]
12710    fn pick_successor_skips_an_agent_another_seat_holds() {
12711        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
12712        let roster = [spec("a"), spec("b"), spec("c")];
12713        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12714        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12715    }
12716
12717    #[test]
12718    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12719        // The other seat started on c but was handed to d; c is free again.
12720        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12721        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12722        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12723        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12724        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12725    }
12726
12727    #[test]
12728    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12729        let roster = [spec("a"), spec("b")];
12730        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12731        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12732        let carried = ids(&["b"]);
12733        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12734        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12735    }
12736
12737    #[test]
12738    fn pick_successor_returns_none_without_any_untried_successor() {
12739        let roster = [spec("a"), spec("b")];
12740        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12741        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12742        let carried = ids(&["a"]);
12743        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12744    }
12745
12746    #[test]
12747    fn pick_successor_rescue_avoids_another_seats_occupant() {
12748        // b is a carried failure and free; a is held by the other seat.
12749        let roster = [spec("a"), spec("b"), spec("c")];
12750        let carried = ids(&["b", "c"]);
12751        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12752        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12753    }
12754
12755    #[test]
12756    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12757        let roster = [spec("a"), spec("b"), spec("c")];
12758        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12759        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12760    }
12761
12762    #[test]
12763    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12764        let roster = [spec("a"), spec("b"), spec("c")];
12765        let failed = ids(&["a", "b", "c"]);
12766        // Rescue looks at the whole roster, once per id, then runs out.
12767        let mut tried = ids(&["b"]);
12768        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12769        assert_eq!(first.id, "a");
12770        tried.insert(first.id.clone());
12771        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12772        assert_eq!(second.id, "c");
12773        tried.insert(second.id.clone());
12774        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12775    }
12776
12777    #[test]
12778    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12779        let roster = [spec("a"), spec("b")];
12780        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12781        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12782    }
12783
12784    #[test]
12785    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12786        let roster = [spec("a"), spec("b"), spec("c")];
12787        let h = SeatHistory {
12788            failed: ids(&["a"]),
12789            last_ok: Some("b".to_owned()),
12790            last_fail: None,
12791        };
12792        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12793        // A last answerer that left the roster, or later failed, is ignored.
12794        let gone = SeatHistory {
12795            last_ok: Some("zzz".to_owned()),
12796            ..h.clone()
12797        };
12798        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12799        let failed = SeatHistory {
12800            failed: ids(&["a", "b"]),
12801            last_ok: Some("b".to_owned()),
12802            last_fail: None,
12803        };
12804        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12805    }
12806
12807    #[test]
12808    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12809        let first = SeatState::new("review-1", "alpha", 7);
12810        let next = handover_seat("review-1", "gamma", 7);
12811        assert_ne!(first.claude_session, next.claude_session);
12812    }
12813
12814    #[test]
12815    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12816        let mut state = state_with_summary("x", "y");
12817        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12818        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12819        assert_ne!(a.claude_session, b.claude_session);
12820    }
12821
12822    #[test]
12823    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12824        let roster = [spec("a"), spec("b")];
12825        let h = SeatHistory {
12826            failed: ids(&["a", "b"]),
12827            ..SeatHistory::default()
12828        };
12829        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12830        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12831        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12832    }
12833
12834    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12835    // fallback loop depends on to terminate: it must walk forward from the
12836    // seat's own position, never restart at the front of the roster, and it
12837    // must never hand back an id already tried, however many times that id
12838    // happens to appear.
12839
12840    #[test]
12841    fn failure_signature_ignores_numbers_and_paths() {
12842        assert_eq!(
12843            failure_signature("exited with Some(2) and no usable output"),
12844            failure_signature("exited with Some(137) and no usable output")
12845        );
12846        assert_eq!(
12847            failure_signature("process exited with no usable output (exit code 2)"),
12848            failure_signature("process exited with no usable output (exit code 137)")
12849        );
12850        for (old, new) in [
12851            (
12852                "exited with Some(2) and no usable output",
12853                "process exited with no usable output (exit code 2)",
12854            ),
12855            (
12856                "exited with None and no usable output",
12857                "process exited with no usable output (exit code unavailable)",
12858            ),
12859        ] {
12860            let previous = FailClass::Other(failure_signature(old));
12861            let current = FailClass::Other(failure_signature(new));
12862            assert_eq!(previous, current, "resumption retains the failure class");
12863            assert!(!should_hand_over(Some(&previous), &current));
12864        }
12865        assert_eq!(
12866            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12867            failure_signature("cannot open /var/x.txt: denied")
12868        );
12869        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12870    }
12871
12872    #[test]
12873    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12874        let other = FailClass::Other("x".into());
12875        assert!(should_hand_over(None, &FailClass::Quota));
12876        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12877        assert!(should_hand_over(
12878            Some(&FailClass::Timeout),
12879            &FailClass::Timeout
12880        ));
12881        assert!(should_hand_over(None, &other));
12882        assert!(!should_hand_over(Some(&other), &other));
12883        assert!(should_hand_over(
12884            Some(&other),
12885            &FailClass::Other("y".into())
12886        ));
12887        // A quota or timeout in between ends the run of identical failures.
12888        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12889        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12890    }
12891
12892    #[test]
12893    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12894        let a = SeatState::new("judge-1", "alpha", 7);
12895        let b = handover_seat("judge-1", "beta", 7);
12896        assert_ne!(a.claude_session, b.claude_session);
12897        assert_eq!(b.turns, 0);
12898    }
12899
12900    #[test]
12901    fn a_timeout_is_classified_apart_from_other_failures() {
12902        assert_eq!(
12903            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12904            Some(FailClass::Timeout)
12905        );
12906        assert!(matches!(
12907            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12908            Some(FailClass::Other(_))
12909        ));
12910    }
12911
12912    #[test]
12913    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12914        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12915        let tried = BTreeSet::from(["beta".to_owned()]);
12916        // beta sits at index 1; the next candidate is gamma, never alpha —
12917        // which is very likely a different candidate slot's own agent.
12918        let next = next_untried_in_roster(&roster, 1, &tried);
12919        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12920    }
12921
12922    #[test]
12923    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12924        let roster = vec![spec("alpha"), spec("beta")];
12925        let tried = BTreeSet::from(["beta".to_owned()]);
12926        // beta is the roster's last entry: nothing follows it, and alpha —
12927        // earlier in the roster, almost certainly a different candidate
12928        // slot's own agent — must not be reached by wrapping back to it.
12929        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12930    }
12931
12932    #[test]
12933    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12934        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12935        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12936        // beta (index 1) and gamma (index 2, the only entry after it) have
12937        // both been tried; alpha (index 0) never has, but it comes before
12938        // beta's own position, so there is nothing further for this seat.
12939        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12940    }
12941
12942    #[test]
12943    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12944        let roster = vec![spec("a"), spec("a"), spec("b")];
12945        let tried = BTreeSet::from(["a".to_owned()]);
12946        let next = next_untried_in_roster(&roster, 0, &tried);
12947        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12948    }
12949
12950    #[test]
12951    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12952        let roster = vec![spec("a"), spec("b")];
12953        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12954        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12955    }
12956
12957    #[test]
12958    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12959        let dir = tempfile::tempdir().unwrap();
12960        let bay = dir.path().join("ffff");
12961
12962        // Not there yet: nothing to do, nothing to panic on.
12963        remove_if_empty(&bay);
12964        assert!(!bay.exists());
12965
12966        // Something still inside - the winner's worktree, or a stray file -
12967        // keeps the directory standing.
12968        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12969        remove_if_empty(&bay);
12970        assert!(bay.exists(), "non-empty directory must survive");
12971
12972        // Once the last entry is gone, so is the directory itself.
12973        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12974        remove_if_empty(&bay);
12975        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12976    }
12977
12978    // `round_is_clean` is the exact decision this task fixed: a round with a
12979    // seat that never answered must not read the same as a round every seat
12980    // actually reviewed. These are deterministic and process-free by design —
12981    // the equivalent end-to-end check (a real reviewer timing out under a
12982    // live graph run) is a genuine race against wall-clock contention, and a
12983    // spawn slow enough to blow even a generous budget under a loaded test
12984    // run must not turn this specific regression check flaky.
12985
12986    #[test]
12987    fn a_full_panel_that_found_nothing_is_clean() {
12988        assert!(round_is_clean(
12989            0,
12990            true,
12991            2,
12992            2,
12993            0,
12994            IncompleteReviewPolicy::Block
12995        ));
12996    }
12997
12998    #[test]
12999    fn a_missing_seat_is_never_clean_under_the_default_policy() {
13000        assert!(!round_is_clean(
13001            0,
13002            true,
13003            1,
13004            2,
13005            0,
13006            IncompleteReviewPolicy::Block
13007        ));
13008    }
13009
13010    #[test]
13011    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13012        assert!(!round_is_clean(
13013            1,
13014            true,
13015            1,
13016            2,
13017            0,
13018            IncompleteReviewPolicy::Warn
13019        ));
13020    }
13021
13022    #[test]
13023    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13024        assert!(round_is_clean(
13025            0,
13026            true,
13027            1,
13028            2,
13029            0,
13030            IncompleteReviewPolicy::Warn
13031        ));
13032    }
13033
13034    #[test]
13035    fn a_full_panel_with_an_open_finding_is_not_clean() {
13036        assert!(!round_is_clean(
13037            1,
13038            true,
13039            2,
13040            2,
13041            0,
13042            IncompleteReviewPolicy::Block
13043        ));
13044    }
13045
13046    #[test]
13047    fn a_full_panel_with_a_red_e2e_is_not_clean() {
13048        assert!(!round_is_clean(
13049            0,
13050            false,
13051            2,
13052            2,
13053            0,
13054            IncompleteReviewPolicy::Block
13055        ));
13056    }
13057
13058    // The stall this task closes: under the default `block` policy, a seat
13059    // missing only because it was rate limited must not force a wait for a
13060    // session limit that will not lift by the next round. `round_is_clean`
13061    // is where that quorum carve-out lives; the review loop around it never
13062    // changes what a reviewer's vote or a finding's severity means.
13063
13064    #[test]
13065    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13066        // 1 of 2 answered, and the one missing was quota'd — the exact
13067        // "review-2 rate limited (quota)" shape from the field report.
13068        assert!(round_is_clean(
13069            0,
13070            true,
13071            1,
13072            2,
13073            1,
13074            IncompleteReviewPolicy::Block
13075        ));
13076    }
13077
13078    #[test]
13079    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13080        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
13081        // not a quota loss (`quota_missing` stays 0) — worth another try.
13082        assert!(!round_is_clean(
13083            0,
13084            true,
13085            1,
13086            2,
13087            0,
13088            IncompleteReviewPolicy::Block
13089        ));
13090    }
13091
13092    #[test]
13093    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
13094        assert!(!round_is_clean(
13095            1,
13096            true,
13097            1,
13098            2,
13099            1,
13100            IncompleteReviewPolicy::Block
13101        ));
13102        assert!(!round_is_clean(
13103            0,
13104            false,
13105            1,
13106            2,
13107            1,
13108            IncompleteReviewPolicy::Block
13109        ));
13110    }
13111
13112    #[test]
13113    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
13114        // Every seat quota'd, nobody answered: there is no panel to decide
13115        // on, so this must fall through to the existing block-and-retry
13116        // fallback rather than call an unreviewed patch clean.
13117        assert!(!round_is_clean(
13118            0,
13119            true,
13120            0,
13121            2,
13122            2,
13123            IncompleteReviewPolicy::Block
13124        ));
13125    }
13126
13127    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
13128        CommandOutcome {
13129            command: "test".to_owned(),
13130            code,
13131            output_tail: String::new(),
13132            duration_ms: 0,
13133            resource_blocked,
13134        }
13135    }
13136
13137    #[test]
13138    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
13139        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
13140        assert!(
13141            !verify_inconclusive(&[outcome(Some(1), false)]),
13142            "an ordinary failure is still evidence about the patch"
13143        );
13144        assert!(verify_inconclusive(&[outcome(None, true)]));
13145        assert!(
13146            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
13147            "one inconclusive outcome taints the whole batch"
13148        );
13149        assert!(!verify_inconclusive(&[]));
13150    }
13151
13152    #[tokio::test]
13153    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
13154        // Alive for the first two checks, then dead - confirms the loop
13155        // actually re-polls rather than deciding once and sleeping out the
13156        // ceiling regardless.
13157        let calls = std::sync::atomic::AtomicUsize::new(0);
13158        let started = Instant::now();
13159        wait_for_pids_with(
13160            &[123],
13161            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
13162            Duration::from_millis(5),
13163            Duration::from_secs(5),
13164        )
13165        .await;
13166        assert!(
13167            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
13168            "must keep checking rather than deciding on the first answer"
13169        );
13170        assert!(
13171            started.elapsed() < Duration::from_secs(1),
13172            "must return the moment it is confirmed dead, not wait out the ceiling"
13173        );
13174    }
13175
13176    #[tokio::test]
13177    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
13178        let started = Instant::now();
13179        wait_for_pids_with(
13180            &[123],
13181            |_| true, // never reports dead
13182            Duration::from_millis(5),
13183            Duration::from_millis(30),
13184        )
13185        .await;
13186        let elapsed = started.elapsed();
13187        assert!(
13188            elapsed >= Duration::from_millis(30),
13189            "must not give up before its own ceiling: {elapsed:?}"
13190        );
13191        assert!(
13192            elapsed < Duration::from_secs(1),
13193            "must not wait past its own ceiling either: {elapsed:?}"
13194        );
13195    }
13196
13197    #[tokio::test]
13198    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
13199        let started = Instant::now();
13200        wait_for_pids_with(
13201            &[],
13202            |_| true,
13203            Duration::from_secs(5),
13204            Duration::from_secs(5),
13205        )
13206        .await;
13207        assert!(
13208            started.elapsed() < Duration::from_millis(200),
13209            "an empty pid list has nothing to confirm"
13210        );
13211    }
13212
13213    // `review_conclusion` is the exact decision the review hand-off task
13214    // fixed: a round budget spent (or a tree that stopped moving) must not
13215    // collapse into `Blocked` regardless of what verification actually
13216    // said. Deterministic and process-free for the same reason the
13217    // `round_is_clean` family above is.
13218    fn review_round(
13219        clean: bool,
13220        blocking: usize,
13221        answered: usize,
13222        expected: usize,
13223        progressed: bool,
13224        e2e_ok: bool,
13225    ) -> ReviewRound {
13226        ReviewRound {
13227            round: 1,
13228            head: "h".to_owned(),
13229            verified_head: None,
13230            verified_at: None,
13231            reviews: Vec::new(),
13232            e2e: vec![CommandOutcome {
13233                command: "test".to_owned(),
13234                code: Some(if e2e_ok { 0 } else { 1 }),
13235                output_tail: String::new(),
13236                duration_ms: 0,
13237                resource_blocked: false,
13238            }],
13239            verify_retried: false,
13240            e2e_deferred: false,
13241            e2e_defer_reason: None,
13242            fix: None,
13243            blocking,
13244            answered,
13245            expected,
13246            clean,
13247            progressed,
13248            vote_split: false,
13249            reconsideration: Vec::new(),
13250            verdict: None,
13251        }
13252    }
13253
13254    #[test]
13255    fn review_conclusion_is_none_when_nothing_has_run() {
13256        assert_eq!(review_conclusion(&[], 3), None);
13257    }
13258
13259    #[test]
13260    fn review_conclusion_is_none_while_rounds_remain() {
13261        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
13262        assert_eq!(review_conclusion(&rounds, 3), None);
13263    }
13264
13265    #[test]
13266    fn review_conclusion_is_gating_once_a_round_is_clean() {
13267        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
13268        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
13269    }
13270
13271    #[test]
13272    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
13273        let rounds = vec![
13274            review_round(false, 1, 2, 2, true, true),
13275            review_round(false, 1, 2, 2, true, true),
13276        ];
13277        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
13278    }
13279
13280    #[test]
13281    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
13282        let rounds = vec![
13283            review_round(false, 1, 2, 2, true, true),
13284            review_round(false, 1, 2, 2, true, false),
13285        ];
13286        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
13287    }
13288
13289    #[test]
13290    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
13291        // Magi never got a command to run against this round's own head — a
13292        // resource-blocked attempt, not a red one — so this must never
13293        // settle on `Blocked` the way a genuine e2e failure would. `None`
13294        // here is what tells `Runner::review_loop` to retry the check
13295        // itself rather than trust this cheap recomputation with a verdict
13296        // it cannot actually produce.
13297        let mut blocked = review_round(false, 1, 2, 2, true, false);
13298        blocked.e2e[0].resource_blocked = true;
13299        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
13300        assert_eq!(review_conclusion(&rounds, 2), None);
13301    }
13302
13303    #[test]
13304    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
13305        // Missing input, not a verified tree — never a hand-off candidate.
13306        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
13307        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
13308    }
13309
13310    #[test]
13311    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
13312        let rounds = vec![
13313            review_round(false, 1, 2, 2, false, true),
13314            review_round(false, 1, 2, 2, false, true),
13315        ];
13316        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
13317    }
13318
13319    fn secs(n: u64) -> Duration {
13320        Duration::from_secs(n)
13321    }
13322
13323    /// A throwaway repo with one commit on `main`, for tests that need `merge`
13324    /// to make real (and, if it runs at all, real*ly fail*) git calls.
13325    #[tokio::test]
13326    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13327        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
13328        if std::env::var_os(CHILD).is_none() {
13329            // Other tests reconcile every run in the process-global home,
13330            // briefly taking owner locks. This ancestry test expects an idle
13331            // predecessor, so give it its own process and pinned test home.
13332            let output = std::process::Command::new(std::env::current_exe().unwrap())
13333                .args([
13334                    "--exact",
13335                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
13336                    "--nocapture",
13337                ])
13338                .env(CHILD, "1")
13339                .env_remove("MAGI_WORKER_HOME")
13340                .env_remove("MAGI_WORKER_ID")
13341                .output()
13342                .unwrap();
13343            assert!(
13344                output.status.success()
13345                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
13346                "{}\n{}",
13347                String::from_utf8_lossy(&output.stdout),
13348                String::from_utf8_lossy(&output.stderr)
13349            );
13350            return;
13351        }
13352        ask_test_home();
13353        let tmp = tempfile::tempdir().unwrap();
13354        let repo = tmp.path().join("repo");
13355        std::fs::create_dir_all(&repo).unwrap();
13356        init_repo(&repo);
13357        let g = |args: &[&str]| {
13358            let out = std::process::Command::new("git")
13359                .args(args)
13360                .current_dir(&repo)
13361                .quiet()
13362                .output()
13363                .unwrap();
13364            assert!(
13365                out.status.success(),
13366                "{}",
13367                String::from_utf8_lossy(&out.stderr)
13368            );
13369        };
13370        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13371        g(&["checkout", "-b", "historical"]);
13372        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13373        g(&["add", "."]);
13374        g(&["commit", "-m", "old alternative"]);
13375        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13376        g(&["checkout", "-b", "magi/task-selected", &base]);
13377        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13378        g(&["add", "."]);
13379        g(&["commit", "-m", "measurement infrastructure"]);
13380        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13381        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13382        g(&["add", "."]);
13383        g(&["commit", "-m", "dependent implementation"]);
13384        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13385        g(&["checkout", "main"]);
13386        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13387        g(&["add", "."]);
13388        g(&["commit", "-m", "unrelated base change"]);
13389        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13390        let origin = tmp.path().join("origin.git");
13391        g(&[
13392            "clone",
13393            "--bare",
13394            repo.to_str().unwrap(),
13395            origin.to_str().unwrap(),
13396        ]);
13397        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13398        g(&["fetch", "origin"]);
13399        let selection = crate::pr_recovery::Selection {
13400            url: "https://example.test/project/pull/1".into(),
13401            branch: "magi/task-selected".into(),
13402            head: head.clone(),
13403            source_head: None,
13404            source_run: None,
13405            source_proof: None,
13406            disposition: Some(crate::pr_recovery::Disposition::Review),
13407            instruction: "preserve measured behavior".into(),
13408            approved_at: jiff::Timestamp::now(),
13409        };
13410        let mut template = runner_at(RunStatus::Reviewing);
13411        template.state.repo = repo.clone();
13412        template.state.base_commit = current.clone();
13413        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13414        template.state.origin = Some(crate::run::Origin::queue("selected"));
13415        template.state.pr_head = Some(selection.branch.clone());
13416        template.state.pr_recovery = Some(selection.clone());
13417        let mut runner =
13418            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13419                .await
13420                .unwrap();
13421        let wt = runner.state.winner().unwrap().worktree.clone();
13422        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13423        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13424        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13425        assert!(runner.state.seeds.is_empty());
13426        assert_eq!(runner.state.instruction, "preserve measured behavior");
13427        runner.sync_to_base().await.unwrap();
13428        assert_eq!(
13429            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13430            "prerequisite"
13431        );
13432        assert_eq!(
13433            std::fs::read_to_string(wt.join("implementation")).unwrap(),
13434            "measured implementation"
13435        );
13436        assert_eq!(
13437            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13438            "current main change"
13439        );
13440        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13441        assert_eq!(
13442            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13443            head
13444        );
13445
13446        // A failed selected-source setup stays inspectable, never resumable as a new competition.
13447        let mut partial = runner_at(RunStatus::Reviewing);
13448        partial.state.repo = repo.clone();
13449        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
13450        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
13451            run: Some("exit 1".into()),
13452            ..Default::default()
13453        }];
13454        let selection = runner.state.pr_recovery.clone().unwrap();
13455        partial.state.pr_recovery = Some(selection.clone());
13456        partial.state.pr_head = Some(selection.branch.clone());
13457        partial.state.origin = Some(crate::run::Origin::queue("selected"));
13458        let id = partial.state.id.clone();
13459        let error = match Runner::open_selected(
13460            &repo,
13461            partial.state,
13462            partial.roles,
13463            1,
13464            runner.state.base_commit.clone(),
13465            selection,
13466        )
13467        .await
13468        {
13469            Ok(_) => panic!("setup must fail"),
13470            Err(e) => e,
13471        };
13472        assert_eq!(
13473            error
13474                .downcast_ref::<crate::pr_recovery::StartupFailed>()
13475                .unwrap()
13476                .run,
13477            id
13478        );
13479        let failed = RunState::load(&id).unwrap();
13480        assert_eq!(failed.status, RunStatus::Blocked);
13481        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
13482        assert!(Runner::resume(&id).is_err());
13483        runner.state.status = RunStatus::Ready;
13484        runner.state.driver_exited = true;
13485        runner.state.save().unwrap();
13486        let mut retry = runner_at(RunStatus::Reviewing);
13487        retry.state.repo = repo.clone();
13488        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
13489        let mut reselected = runner.state.pr_recovery.clone().unwrap();
13490        reselected.approved_at = jiff::Timestamp::now();
13491        retry.state.origin = Some(crate::run::Origin::queue("selected"));
13492        retry.state.pr_head = Some(reselected.branch.clone());
13493        retry.state.pr_recovery = Some(reselected.clone());
13494        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
13495            .await
13496            .unwrap();
13497        let restarted = Runner::open_selected(
13498            &repo,
13499            retry.state,
13500            retry.roles,
13501            1,
13502            runner.state.base_commit.clone(),
13503            reselected,
13504        )
13505        .await
13506        .unwrap();
13507        assert_eq!(
13508            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
13509                .await
13510                .unwrap(),
13511            head
13512        );
13513        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13514        assert_eq!(
13515            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13516            head
13517        );
13518    }
13519
13520    #[tokio::test]
13521    #[cfg(unix)]
13522    async fn fallback_and_interrupted_restoration_require_coordination() {
13523        use std::os::fd::AsRawFd as _;
13524        ask_test_home();
13525        let tmp = tempfile::tempdir().unwrap();
13526        let mut runner = registered_runner(tmp.path()).await;
13527        runner.state.config.graph.timeout_resource_wait = 1;
13528        let winner = runner.state.candidates[0].clone();
13529        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13530        let leftover = winner.worktree.join("leftover");
13531        std::fs::write(&leftover, "untrusted edits").unwrap();
13532        let next = tmp.path().join("next-fixer");
13533        let mut failed = spec("first");
13534        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
13535        let mut successor = spec("second");
13536        successor.command = vec![
13537            "sh".into(),
13538            "-c".into(),
13539            format!("touch '{}'; echo done", next.display()),
13540        ];
13541        let attempts = vec![
13542            (failed.clone(), "first".into()),
13543            (successor.clone(), "second".into()),
13544        ];
13545        let lock_path = runner.state.repo.join(".git/updater.lock");
13546        let external = std::fs::File::create(&lock_path).unwrap();
13547        assert_eq!(
13548            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13549            0
13550        );
13551        let artifacts = tmp.path().join("artifacts");
13552        let build = |spec, seat| SeatJob {
13553            spec,
13554            seat,
13555            cwd: winner.worktree.clone(),
13556            prompt: "fixture".into(),
13557            timeout: Duration::from_secs(10),
13558            allow_write: true,
13559            sessions: false,
13560            artifacts: artifacts.clone(),
13561            stem: "fix".into(),
13562            handover: None,
13563        };
13564        let (_, _, out) = runner
13565            .ask_fixer_among(
13566                attempts.clone(),
13567                "graft",
13568                None,
13569                Some((&winner.worktree, &head)),
13570                build,
13571            )
13572            .await;
13573        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13574        assert!(leftover.exists(), "contention must prevent clean");
13575        assert!(!next.exists());
13576        assert!(runner.state.handovers.is_empty());
13577        // Early interrupted-repair rollback uses the same protected route.
13578        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13579            from_run: "earlier".into(),
13580            generation: 1,
13581            mode: crate::recovery::Mode::VerifyRepair,
13582            reasons: Vec::new(),
13583            reviewed_head: None,
13584            rounds: Vec::new(),
13585            open_findings: Vec::new(),
13586            evidence: None,
13587            criteria_inherited: true,
13588            repair: crate::recovery::Repair::Started {
13589                before: head.clone(),
13590            },
13591        });
13592        assert!(
13593            runner
13594                .recovery_repair(&winner)
13595                .await
13596                .unwrap_err()
13597                .is::<crate::repo_lock::ResourceBlocked>()
13598        );
13599        assert!(matches!(
13600            runner.state.recovery.as_ref().unwrap().repair,
13601            crate::recovery::Repair::Started { .. }
13602        ));
13603        assert!(leftover.exists());
13604        drop(external);
13605        runner.recovery_repair(&winner).await.unwrap();
13606        assert!(!leftover.exists());
13607        assert!(matches!(
13608            runner.state.recovery.as_ref().unwrap().repair,
13609            crate::recovery::Repair::Done { .. }
13610        ));
13611        // A hard lock-open failure also leaves HEAD and untracked work intact.
13612        std::fs::remove_file(&lock_path).unwrap();
13613        std::fs::create_dir(&lock_path).unwrap();
13614        std::fs::write(&leftover, "untrusted edits").unwrap();
13615        let (_, _, out) = runner
13616            .ask_fixer_among(
13617                attempts,
13618                "graft",
13619                None,
13620                Some((&winner.worktree, &head)),
13621                build,
13622            )
13623            .await;
13624        assert!(matches!(out, AgentOutcome::Failed(_)));
13625        assert!(leftover.exists());
13626        assert!(!next.exists());
13627        assert_eq!(
13628            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13629            head
13630        );
13631    }
13632
13633    #[cfg(unix)]
13634    async fn registered_runner(root: &Path) -> Runner {
13635        ask_test_home();
13636        let repo = root.join("repo");
13637        std::fs::create_dir_all(&repo).unwrap();
13638        init_repo(&repo);
13639        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13640        let mut runner = runner_at(RunStatus::Prep);
13641        runner.state.repo = repo.clone();
13642        runner.state.config.coordination = crate::config::test_coordination();
13643        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13644        runner.state.config.graph.worktree_root = Some(root.join("wt"));
13645        runner.state.config.graph.advise = false;
13646        runner.roles.implementers = vec![spec("implement")];
13647        runner.roles.judges = vec![spec("judge")];
13648        runner.prep().await.unwrap();
13649        runner
13650    }
13651
13652    #[tokio::test]
13653    #[cfg(unix)]
13654    async fn parked_implementer_persists_completed_work() {
13655        for registered in [false, true] {
13656            let tmp = tempfile::tempdir().unwrap();
13657            let mut runner = registered_runner(tmp.path()).await;
13658            if !registered {
13659                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13660            }
13661            let marker = tmp.path().join("started");
13662            let release = tmp.path().join("release");
13663            let mut implementer = spec("implement");
13664            implementer.command = vec![
13665                "sh".into(),
13666                "-c".into(),
13667                format!(
13668                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13669                    marker.display(),
13670                    release.display()
13671                ),
13672            ];
13673            runner.roles.implementers = vec![implementer.clone()];
13674            runner.state.config.agents = vec![implementer];
13675            let pause = runner.pause.clone();
13676            let (result, ()) = tokio::join!(runner.implement(), async {
13677                tokio::time::timeout(Duration::from_secs(10), async {
13678                    while !marker.exists() {
13679                        tokio::time::sleep(Duration::from_millis(10)).await;
13680                    }
13681                })
13682                .await
13683                .unwrap();
13684                pause.park();
13685                std::fs::write(&release, "go").unwrap();
13686            });
13687            result.unwrap();
13688            let candidate = &runner.state.candidates[0];
13689            assert!(candidate.commits > 0, "{candidate:?}");
13690            assert!(candidate.summary.contains("saved-result"));
13691            assert_eq!(candidate.files, 1);
13692            let before = candidate.commits;
13693            runner.implement().await.unwrap();
13694            assert_eq!(runner.state.candidates[0].commits, before);
13695        }
13696    }
13697
13698    #[tokio::test]
13699    #[cfg(unix)]
13700    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13701        let tmp = tempfile::tempdir().unwrap();
13702        let mut runner = registered_runner(tmp.path()).await;
13703        let winner = runner.state.candidates[0].clone();
13704        let cache = tmp.path().join("cache");
13705        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13706        runner.state.config.graph.timeout_resource_wait = 0;
13707        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13708        let crate::cache::AcquireOutcome::Acquired(held) =
13709            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13710        else {
13711            panic!("cache must be free")
13712        };
13713        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13714            from_run: "earlier".into(),
13715            generation: 1,
13716            mode: crate::recovery::Mode::VerifyRepair,
13717            reasons: Vec::new(),
13718            reviewed_head: None,
13719            rounds: Vec::new(),
13720            open_findings: Vec::new(),
13721            evidence: None,
13722            criteria_inherited: true,
13723            repair: crate::recovery::Repair::NotStarted,
13724        });
13725        runner.recovery_repair(&winner).await.unwrap();
13726        assert!(matches!(
13727            runner.state.recovery.as_ref().unwrap().repair,
13728            crate::recovery::Repair::WaitingForRestore { .. }
13729        ));
13730        drop(held);
13731        runner.state.resource_block = None;
13732        runner.state.config.graph.timeout_resource_wait = 1;
13733        runner.recovery_repair(&winner).await.unwrap();
13734        assert!(matches!(
13735            runner.state.recovery.as_ref().unwrap().repair,
13736            crate::recovery::Repair::Done { .. }
13737        ));
13738        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13739    }
13740
13741    #[tokio::test]
13742    #[cfg(unix)]
13743    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13744        use std::os::fd::AsRawFd as _;
13745        for graft in [false, true] {
13746            let tmp = tempfile::tempdir().unwrap();
13747            let mut runner = registered_runner(tmp.path()).await;
13748            runner.state.config.graph.timeout_resource_wait = 1;
13749            let winner = runner.state.candidates[0].clone();
13750            let marker = tmp.path().join("fixer-started");
13751            let mut fixer = spec("fixer");
13752            fixer.command = vec![
13753                "sh".into(),
13754                "-c".into(),
13755                format!("echo started >> '{}'; echo done", marker.display()),
13756            ];
13757            runner.roles.fixer = Some(vec![fixer]);
13758            if graft {
13759                let mut loser = winner.clone();
13760                loser.label = 'B';
13761                loser.index = 1;
13762                runner.state.candidates.push(loser);
13763                runner.state.config.graph.graft_runner_up = true;
13764                runner.state.tally = Some(Tally {
13765                    first_choice: BTreeMap::new(),
13766                    borda: BTreeMap::new(),
13767                    winner: winner.label,
13768                    rankings: 1,
13769                    unanimous_initial: true,
13770                    deliberated: false,
13771                    changed_votes: 0,
13772                    unanimous_final: true,
13773                    tie_break: None,
13774                    judges: 1,
13775                    present: 1,
13776                    quorum: 1,
13777                    met_quorum: true,
13778                    uncontested: None,
13779                });
13780                runner.state.judgements.push(crate::run::Judgement {
13781                    judge: 1,
13782                    seat: "judge-1".into(),
13783                    agent: "judge".into(),
13784                    ranking: vec![winner.label, 'B'],
13785                    reasons: BTreeMap::new(),
13786                    confidence: None,
13787                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13788                    order: vec![0, 1],
13789                    failed: None,
13790                    duration_ms: 0,
13791                });
13792            } else {
13793                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13794                    from_run: "earlier".into(),
13795                    generation: 1,
13796                    mode: crate::recovery::Mode::VerifyRepair,
13797                    reasons: Vec::new(),
13798                    reviewed_head: None,
13799                    rounds: Vec::new(),
13800                    open_findings: Vec::new(),
13801                    evidence: None,
13802                    criteria_inherited: true,
13803                    repair: crate::recovery::Repair::NotStarted,
13804                });
13805            }
13806            let reader = crate::repo_lock::SeatLease::for_checkout(
13807                &winner.worktree,
13808                false,
13809                crate::config::test_coordination().as_ref(),
13810            )
13811            .await
13812            .unwrap();
13813            let pause = runner.pause.clone();
13814            let (result, ()) = tokio::join!(
13815                async {
13816                    if graft {
13817                        runner.graft().await
13818                    } else {
13819                        runner.recovery_repair(&winner).await
13820                    }
13821                },
13822                async {
13823                    tokio::time::sleep(Duration::from_millis(100)).await;
13824                    pause.park();
13825                }
13826            );
13827            result.unwrap();
13828            assert!(!marker.exists(), "a blocked seat must not spawn");
13829            // Exercise the persisted state, as a binary replacement/resume does.
13830            runner.state = RunState::load(&runner.state.id).unwrap();
13831            if graft {
13832                assert_eq!(
13833                    runner.state.graft.as_ref().unwrap().outcome,
13834                    GraftOutcome::WaitingForRestore
13835                );
13836            } else {
13837                assert!(matches!(
13838                    runner.state.recovery.as_ref().unwrap().repair,
13839                    crate::recovery::Repair::WaitingForRestore { .. }
13840                ));
13841            }
13842            drop(reader);
13843            runner.pause = Pause::new();
13844            runner.state.resource_block = None;
13845            // Deferred restoration must still respect an external lock owner.
13846            let external =
13847                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13848            assert_eq!(
13849                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13850                0
13851            );
13852            let leftover = winner.worktree.join("leftover");
13853            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13854            let result = if graft {
13855                runner.graft().await
13856            } else {
13857                runner.recovery_repair(&winner).await
13858            };
13859            assert!(
13860                result
13861                    .unwrap_err()
13862                    .is::<crate::repo_lock::ResourceBlocked>()
13863            );
13864            assert!(leftover.exists());
13865            assert!(!marker.exists());
13866            drop(external);
13867            runner.state.resource_block = None;
13868            let result = if graft {
13869                runner.graft().await
13870            } else {
13871                runner.recovery_repair(&winner).await
13872            };
13873            result.unwrap();
13874            assert!(!leftover.exists(), "restore must precede retry");
13875            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13876            if graft {
13877                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13878            } else {
13879                assert!(matches!(
13880                    runner.state.recovery.as_ref().unwrap().repair,
13881                    crate::recovery::Repair::Done { .. }
13882                ));
13883            }
13884        }
13885    }
13886
13887    #[tokio::test]
13888    #[cfg(unix)]
13889    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13890        use std::os::fd::AsRawFd as _;
13891        let tmp = tempfile::tempdir().unwrap();
13892        let mut runner = registered_runner(tmp.path()).await;
13893        runner.state.config.graph.timeout_resource_wait = 1;
13894        let winner = runner.state.candidates[0].clone();
13895        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13896        runner.state.tally = Some(Tally {
13897            first_choice: BTreeMap::new(),
13898            borda: BTreeMap::new(),
13899            winner: winner.label,
13900            rankings: 0,
13901            unanimous_initial: false,
13902            deliberated: false,
13903            changed_votes: 0,
13904            unanimous_final: false,
13905            tie_break: None,
13906            judges: 0,
13907            present: 0,
13908            quorum: 0,
13909            met_quorum: true,
13910            uncontested: Some("fixture".into()),
13911        });
13912        runner.state.graft = Some(GraftRecord {
13913            pre: pre.clone(),
13914            post: None,
13915            lines: None,
13916            limit: None,
13917            agent: None,
13918            outcome: GraftOutcome::Running,
13919            requested: Vec::new(),
13920            adoption: None,
13921            adoption_error: None,
13922        });
13923        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13924        git::commit_all(&winner.worktree, "unfinished graft")
13925            .await
13926            .unwrap();
13927        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13928        let leftover = winner.worktree.join("leftover");
13929        std::fs::write(&leftover, "untrusted edits").unwrap();
13930        let path = runner.state.repo.join(".git/updater.lock");
13931        let external = std::fs::File::create(&path).unwrap();
13932        assert_eq!(
13933            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13934            0
13935        );
13936        assert!(
13937            runner
13938                .graft()
13939                .await
13940                .unwrap_err()
13941                .is::<crate::repo_lock::ResourceBlocked>()
13942        );
13943        assert!(matches!(
13944            runner.state.graft.as_ref().unwrap().outcome,
13945            GraftOutcome::Running
13946        ));
13947        assert_eq!(
13948            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13949            unfinished
13950        );
13951        assert!(leftover.exists());
13952        drop(external);
13953        std::fs::remove_file(&path).unwrap();
13954        std::fs::create_dir(&path).unwrap();
13955        assert!(runner.graft().await.is_err());
13956        assert!(matches!(
13957            runner.state.graft.as_ref().unwrap().outcome,
13958            GraftOutcome::Running
13959        ));
13960        assert_eq!(
13961            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13962            unfinished
13963        );
13964        assert!(leftover.exists());
13965        std::fs::remove_dir(&path).unwrap();
13966        runner.graft().await.unwrap();
13967        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13968        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13969        assert!(!leftover.exists());
13970    }
13971
13972    #[tokio::test]
13973    #[cfg(unix)]
13974    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13975        use std::os::fd::AsRawFd as _;
13976        let tmp = tempfile::tempdir().unwrap();
13977        let mut runner = registered_runner(tmp.path()).await;
13978        runner.state.prep_complete = Some(false);
13979        runner.state.config.graph.timeout_resource_wait = 1;
13980        runner.state.save().unwrap();
13981        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13982        assert_eq!(
13983            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13984            0
13985        );
13986        runner.execute().await.unwrap();
13987        let disk = RunState::load(&runner.state.id).unwrap();
13988        assert!(disk.driver_exited);
13989        assert_eq!(disk.driver_pid, None);
13990        assert_eq!(runner.state.status, RunStatus::Blocked);
13991        assert_eq!(
13992            runner.state.resource_block.as_ref().unwrap().kind,
13993            "repository coordination"
13994        );
13995        assert!(runner.state.prep_failure.is_none());
13996        assert!(runner.state.seats.is_empty());
13997        runner.pause.park();
13998        // Include headroom for Git subprocesses under parallel test load;
13999        // the repository contention deadline itself remains one second.
14000        tokio::time::timeout(Duration::from_secs(10), runner.execute())
14001            .await
14002            .unwrap()
14003            .unwrap();
14004        assert!(runner.state.parked);
14005        assert!(runner.state.prep_failure.is_none());
14006        let disk = RunState::load(&runner.state.id).unwrap();
14007        assert!(disk.driver_exited);
14008        assert_eq!(disk.driver_pid, None);
14009    }
14010
14011    #[tokio::test]
14012    #[cfg(unix)]
14013    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
14014        use std::os::fd::AsRawFd as _;
14015        let tmp = tempfile::tempdir().unwrap();
14016        let mut runner = registered_runner(tmp.path()).await;
14017        runner.state.prep_complete = Some(false);
14018        runner.state.config.graph.timeout_resource_wait = 30;
14019        runner.state.save().unwrap();
14020        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14021        assert_eq!(
14022            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14023            0
14024        );
14025        assert!(
14026            tokio::time::timeout(Duration::from_millis(100), runner.execute())
14027                .await
14028                .is_err()
14029        );
14030        let disk = RunState::load(&runner.state.id).unwrap();
14031        assert!(disk.driver_exited);
14032        assert_eq!(disk.driver_pid, None);
14033        assert!(disk.status.resumable());
14034        assert!(runner.state.driver_exited);
14035        assert_eq!(runner.state.driver_pid, None);
14036        runner.state.save().unwrap();
14037        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
14038    }
14039
14040    #[tokio::test]
14041    #[cfg(unix)]
14042    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
14043        let tmp = tempfile::tempdir().unwrap();
14044        let mut runner = registered_runner(tmp.path()).await;
14045        let winner = runner.state.candidates[0].clone();
14046        let cache = tmp.path().join("cache");
14047        let started = tmp.path().join("pre-gate-started");
14048        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
14049        runner.state.config.verify.gate =
14050            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
14051        let home = crate::run::home();
14052        let owner =
14053            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
14054        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
14055            crate::cache::AcquireOutcome::Acquired(g) => g,
14056            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
14057        };
14058        let mut gate = Box::pin(runner.run_gate(&winner));
14059        assert!(
14060            tokio::time::timeout(Duration::from_millis(150), &mut gate)
14061                .await
14062                .is_err()
14063        );
14064        assert!(!started.exists(), "pre-gate must wait for the cache too");
14065        // Simulate the wave that already owns the cache entering invoke.
14066        // It must obtain the editor reservation and finish without waiting
14067        // for the gate that is queued behind its cache.
14068        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
14069        let editor = crate::repo_lock::SeatLease::checkout_with(
14070            &winner.worktree,
14071            true,
14072            crate::config::test_coordination().as_ref(),
14073            &wait,
14074        )
14075        .await
14076        .unwrap();
14077        drop(editor);
14078        guard.release();
14079        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14080            .await
14081            .unwrap()
14082            .unwrap();
14083        drop(gate);
14084        assert!(outcomes.iter().all(CommandOutcome::ok));
14085        assert!(started.exists());
14086    }
14087
14088    #[tokio::test]
14089    #[cfg(unix)]
14090    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
14091        use std::os::fd::AsRawFd as _;
14092
14093        ask_test_home();
14094        let tmp = tempfile::tempdir().unwrap();
14095        let repo = tmp.path().join("repo");
14096        std::fs::create_dir_all(&repo).unwrap();
14097        init_repo(&repo);
14098        let common = repo.join(".git");
14099        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14100        let mut runner = runner_at(RunStatus::Prep);
14101        runner.state.repo = repo.clone();
14102        runner.state.config.coordination = crate::config::test_coordination();
14103        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14104        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14105        runner.state.config.graph.advise = false;
14106        runner.roles.implementers = vec![spec("implement")];
14107        runner.roles.judges = vec![spec("judge")];
14108        runner.prep().await.unwrap();
14109        let winner = runner.state.candidates[0].clone();
14110        // Emulate edits already left by a completed seat. The fixture CLI
14111        // only reports completion; it performs no repository operations.
14112        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
14113        let completed = tmp.path().join("seat-completed");
14114        runner.roles.implementers[0].command = vec![
14115            "sh".into(),
14116            "-c".into(),
14117            format!("touch '{}'; echo completed", completed.display()),
14118        ];
14119        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
14120        assert_eq!(
14121            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14122            0
14123        );
14124        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14125        let mut implement = Box::pin(runner.implement());
14126        tokio::select! {
14127            result = &mut implement => panic!("rescue must wait: {result:?}"),
14128            result = tokio::time::timeout(Duration::from_secs(10), async {
14129                while !completed.exists() {
14130                    tokio::time::sleep(Duration::from_millis(20)).await;
14131                }
14132            }) => result.unwrap(),
14133        }
14134        assert!(
14135            tokio::time::timeout(Duration::from_millis(200), &mut implement)
14136                .await
14137                .is_err()
14138        );
14139        assert_eq!(
14140            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14141            before
14142        );
14143        assert_eq!(
14144            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14145            0
14146        );
14147        tokio::time::timeout(Duration::from_secs(10), &mut implement)
14148            .await
14149            .unwrap()
14150            .unwrap();
14151        drop(implement);
14152        assert_ne!(
14153            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14154            before
14155        );
14156        assert!(git::is_clean(&winner.worktree).await.unwrap());
14157
14158        let started = tmp.path().join("pre-gate-started");
14159        let proceed = tmp.path().join("pre-gate-proceed");
14160        let checked = tmp.path().join("gate-ran");
14161        runner.state.config.verify.pre_gate = vec![format!(
14162            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14163            started.display(),
14164            proceed.display()
14165        )];
14166        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
14167        assert_eq!(
14168            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14169            0
14170        );
14171        let mut gate = Box::pin(runner.run_gate(&winner));
14172        assert!(
14173            tokio::time::timeout(Duration::from_millis(200), &mut gate)
14174                .await
14175                .is_err()
14176        );
14177        assert!(!started.exists());
14178        assert!(!checked.exists());
14179        assert!(git::is_clean(&winner.worktree).await.unwrap());
14180        assert_eq!(
14181            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14182            0
14183        );
14184        tokio::select! {
14185            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
14186            result = tokio::time::timeout(Duration::from_secs(10), async {
14187                while !started.exists() {
14188                    tokio::time::sleep(Duration::from_millis(20)).await;
14189                }
14190            }) => result.unwrap(),
14191        }
14192        assert_eq!(
14193            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14194            -1,
14195            "mechanical edits must hold the real lock"
14196        );
14197        std::fs::write(proceed, "ready").unwrap();
14198        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14199            .await
14200            .unwrap()
14201            .unwrap();
14202        drop(gate);
14203        assert!(outcomes.iter().all(CommandOutcome::ok));
14204        assert!(checked.exists());
14205        assert!(runner.state.pre_gate_commit.is_some());
14206        assert!(git::is_clean(&winner.worktree).await.unwrap());
14207        assert_eq!(
14208            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14209            0,
14210            "gate completion must release coordination"
14211        );
14212        drop(outsider);
14213        std::fs::remove_file(common.join("updater.lock")).unwrap();
14214        std::fs::create_dir(common.join("updater.lock")).unwrap();
14215        std::fs::remove_file(&started).unwrap();
14216        std::fs::remove_file(&checked).unwrap();
14217        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14218        let error = runner.run_gate(&winner).await.unwrap_err();
14219        assert!(
14220            error
14221                .to_string()
14222                .contains("acquire mandatory repository lock")
14223        );
14224        assert!(
14225            !started.exists(),
14226            "failed acquisition must prevent pre-gate"
14227        );
14228        assert!(!checked.exists(), "failed acquisition must prevent gate");
14229        assert!(git::is_clean(&winner.worktree).await.unwrap());
14230        assert_eq!(
14231            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14232            head
14233        );
14234    }
14235
14236    #[tokio::test]
14237    #[cfg(unix)]
14238    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
14239        use std::os::fd::AsRawFd as _;
14240        use std::time::Duration;
14241
14242        ask_test_home();
14243        let tmp = tempfile::tempdir().unwrap();
14244        let repo = tmp.path().join("repo");
14245        std::fs::create_dir_all(&repo).unwrap();
14246        init_repo(&repo);
14247        let common = repo.join(".git");
14248        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14249        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
14250        assert_eq!(
14251            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14252            0
14253        );
14254        let mut runner = runner_at(RunStatus::Prep);
14255        runner.state.repo = repo.clone();
14256        runner.state.config.coordination = crate::config::test_coordination();
14257        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14258        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14259        runner.state.config.graph.advise = true;
14260        runner.state.config.graph.advisors = 1;
14261        runner.roles.implementers = vec![spec("implement")];
14262        runner.roles.judges = vec![spec("judge")];
14263        let root = runner.state.worktree_root();
14264        let started = tmp.path().join("setup-started");
14265        let proceed = tmp.path().join("setup-proceed");
14266        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14267            run: Some(format!(
14268                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14269                started.display(),
14270                proceed.display()
14271            )),
14272            ..Default::default()
14273        }];
14274        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
14275        let config = git::git(&repo, &["config", "--local", "--list"])
14276            .await
14277            .unwrap();
14278        let mut prep = Box::pin(runner.prep());
14279        assert!(
14280            tokio::time::timeout(Duration::from_millis(200), &mut prep)
14281                .await
14282                .is_err()
14283        );
14284        assert!(
14285            !root.exists(),
14286            "contention must prevent even checkout creation"
14287        );
14288        assert!(!started.exists(), "contention must prevent setup");
14289        assert_eq!(
14290            git::git(&repo, &["branch", "--list"]).await.unwrap(),
14291            branches
14292        );
14293        assert_eq!(
14294            git::git(&repo, &["config", "--local", "--list"])
14295                .await
14296                .unwrap(),
14297            config
14298        );
14299        drop(lock);
14300        tokio::select! {
14301            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
14302            result = tokio::time::timeout(Duration::from_secs(10), async {
14303                while !started.exists() {
14304                    tokio::time::sleep(Duration::from_millis(20)).await;
14305                }
14306            }) => result.unwrap(),
14307        }
14308        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
14309        assert_eq!(
14310            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14311            -1
14312        );
14313        assert!(
14314            tokio::time::timeout(
14315                Duration::from_millis(100),
14316                crate::repo_lock::SeatLease::for_checkout(
14317                    &repo,
14318                    true,
14319                    crate::config::test_coordination().as_ref()
14320                )
14321            )
14322            .await
14323            .is_err(),
14324            "preparation must also exclude local editors"
14325        );
14326        std::fs::write(proceed, "ready").unwrap();
14327        tokio::time::timeout(Duration::from_secs(10), &mut prep)
14328            .await
14329            .unwrap()
14330            .unwrap();
14331        drop(prep);
14332        assert_eq!(runner.state.prep_complete, Some(true));
14333        assert!(root.join("judge-1").exists());
14334        assert!(root.join("advisor-1").exists());
14335        assert_eq!(
14336            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14337            0
14338        );
14339    }
14340
14341    #[tokio::test]
14342    #[cfg(unix)]
14343    async fn preparation_lock_failure_prevents_repository_changes() {
14344        ask_test_home();
14345        let tmp = tempfile::tempdir().unwrap();
14346        let repo = tmp.path().join("repo");
14347        std::fs::create_dir_all(&repo).unwrap();
14348        init_repo(&repo);
14349        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14350        // A directory at the required lock path makes opening the real lock
14351        // fail, rather than merely exercising an invalid registration.
14352        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
14353        let mut runner = runner_at(RunStatus::Prep);
14354        runner.state.repo = repo.clone();
14355        runner.state.config.coordination = crate::config::test_coordination();
14356        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14357        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14358        runner.roles.implementers = vec![spec("implement")];
14359        let error = runner.prep().await.unwrap_err();
14360        assert!(
14361            error
14362                .to_string()
14363                .contains("acquire mandatory repository lock")
14364        );
14365        assert!(!runner.state.worktree_root().exists());
14366        assert!(runner.state.candidates.is_empty());
14367        assert!(!repo.join(".git/config.worktree").exists());
14368        assert_eq!(
14369            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
14370            "* main"
14371        );
14372    }
14373
14374    #[tokio::test]
14375    async fn preparation_records_creation_intent_before_git_can_fail() {
14376        ask_test_home();
14377        let tmp = tempfile::tempdir().unwrap();
14378        let repo = tmp.path().join("repo");
14379        std::fs::create_dir_all(&repo).unwrap();
14380        init_repo(&repo);
14381        let mut runner = runner_at(RunStatus::Prep);
14382        runner.state.repo = repo.clone();
14383        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14384        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14385        runner.state.config.blind.commit_msg_hook = false;
14386        runner.roles.implementers = vec![spec("one")];
14387        let label = blind::assign_labels(1, runner.state.seed)[0];
14388        let branch = runner.state.branch_for(label);
14389        // A conflicting branch makes Git refuse creation before a candidate
14390        // can be registered. The durable intent must still identify it.
14391        git::git(&repo, &["branch", &branch]).await.unwrap();
14392        assert!(runner.prep().await.is_err());
14393        let saved = RunState::load(&runner.state.id).unwrap();
14394        assert!(saved.candidates.is_empty());
14395        let event = saved
14396            .events
14397            .iter()
14398            .find(|e| e.node == "prep-intent")
14399            .unwrap();
14400        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
14401        assert_eq!(intent["branch"], branch);
14402        assert_eq!(intent["start"], runner.state.base_commit);
14403        assert_eq!(
14404            intent["worktree"],
14405            runner
14406                .state
14407                .worktree_root()
14408                .join(format!("cand-{label}"))
14409                .to_string_lossy()
14410                .as_ref()
14411        );
14412        assert_eq!(intent["picks"], serde_json::json!([]));
14413    }
14414
14415    #[tokio::test]
14416    async fn invalid_pinned_sources_create_no_partial_worktrees() {
14417        ask_test_home();
14418        let tmp = tempfile::tempdir().unwrap();
14419        let repo = tmp.path().join("repo");
14420        std::fs::create_dir_all(&repo).unwrap();
14421        init_repo(&repo);
14422        let mut runner = runner_at(RunStatus::Prep);
14423        runner.state.repo = repo.clone();
14424        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14425        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14426        runner.state.sources.pick_commits.push(refs::Input {
14427            token: "missing".into(),
14428            sha: "0".repeat(40),
14429            provenance: refs::Provenance::ExplicitCli,
14430        });
14431        assert!(runner.prep().await.is_err());
14432        assert!(runner.state.candidates.is_empty());
14433        assert!(!runner.state.worktree_root().exists());
14434    }
14435
14436    #[tokio::test]
14437    async fn prose_and_legacy_inference_never_import_into_candidates() {
14438        ask_test_home();
14439        let tmp = tempfile::tempdir().unwrap();
14440        let repo = tmp.path().join("repo");
14441        std::fs::create_dir_all(&repo).unwrap();
14442        init_repo(&repo);
14443        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14444        git::git(&repo, &["checkout", "-b", "magi/history/A"])
14445            .await
14446            .unwrap();
14447        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
14448        git::git(&repo, &["add", "."]).await.unwrap();
14449        git::git(&repo, &["commit", "-m", "historical implementation"])
14450            .await
14451            .unwrap();
14452        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
14453        git::git(&repo, &["checkout", "main"]).await.unwrap();
14454        for instruction in [
14455            format!("Example: magi/history/A at {source}"),
14456            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
14457            format!("Never import magi/history/A or {source}"),
14458            format!("Conversation answer: I mentioned {source} only as background"),
14459        ] {
14460            let mut runner = runner_at(RunStatus::Prep);
14461            runner.state.repo = repo.clone();
14462            runner.state.base_commit = base.clone();
14463            runner.state.instruction = instruction.clone();
14464            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
14465            runner.state.config.blind.commit_msg_hook = false;
14466            runner.state.seeds = vec![refs::Seed {
14467                token: "magi/history/A".into(),
14468                sha: source.clone(),
14469                kind: refs::SeedKind::Unmerged,
14470                branch: true,
14471                detail: String::new(),
14472            }];
14473            runner.roles.implementers = vec![spec("one")];
14474            assert_eq!(seeded_instruction(&runner.state), instruction);
14475            // Model an old record with inferred seeds but no structured input field.
14476            let mut legacy = serde_json::to_value(&runner.state).unwrap();
14477            legacy.as_object_mut().unwrap().remove("sources");
14478            runner.state = serde_json::from_value(legacy).unwrap();
14479            assert_eq!(runner.state.sources, refs::Sources::default());
14480            runner.state.save().unwrap();
14481            runner.state = RunState::load(&runner.state.id).unwrap();
14482            runner.prep().await.unwrap();
14483            for candidate in &runner.state.candidates {
14484                assert_eq!(
14485                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
14486                    base
14487                );
14488                assert!(!candidate.worktree.join("unwanted").exists());
14489            }
14490        }
14491    }
14492
14493    #[tokio::test]
14494    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
14495        ask_test_home();
14496        let tmp = tempfile::tempdir().unwrap();
14497        let repo = tmp.path().join("repo");
14498        std::fs::create_dir_all(&repo).unwrap();
14499        init_repo(&repo);
14500        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14501        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
14502        std::fs::write(repo.join("referenced"), "required commit").unwrap();
14503        git::git(&repo, &["add", "."]).await.unwrap();
14504        git::git(&repo, &["commit", "-m", "required seed"])
14505            .await
14506            .unwrap();
14507        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
14508        git::git(&repo, &["checkout", "main"]).await.unwrap();
14509        let mut runner = runner_at(RunStatus::Prep);
14510        runner.state.repo = repo.clone();
14511        runner.state.base_commit = base.clone();
14512        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14513        runner.state.config.graph.advise = false;
14514        runner.state.config.blind.commit_msg_hook = false;
14515        runner.state.sources =
14516            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
14517                .await
14518                .unwrap();
14519        runner.roles.implementers = vec![spec("one"), spec("two")];
14520        runner.roles.judges = vec![spec("judge")];
14521        let label = blind::assign_labels(2, runner.state.seed)[0];
14522        let branch = runner.state.branch_for(label);
14523        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14524        git::worktree_add_branch(&repo, &worktree, &branch, &base)
14525            .await
14526            .unwrap();
14527        runner.state.candidates.push(Candidate {
14528            index: 0,
14529            label,
14530            agent: "one".into(),
14531            branch,
14532            worktree,
14533            summary: String::new(),
14534            stat: String::new(),
14535            files: 0,
14536            commits: 0,
14537            empty: false,
14538            failed: None,
14539            verified_noop: None,
14540            duration_ms: 0,
14541            folded: false,
14542        });
14543        runner.state.save().unwrap();
14544        // Reload the checkpoint immediately before its first cherry-pick.
14545        runner.state = RunState::load(&runner.state.id).unwrap();
14546        assert_eq!(runner.state.prep_complete, Some(false));
14547        runner.prep().await.unwrap();
14548        assert_eq!(runner.state.prep_complete, Some(true));
14549        assert_eq!(runner.state.candidates.len(), 2);
14550        for candidate in &runner.state.candidates {
14551            assert_eq!(
14552                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14553                git::tree_of(&repo, &seed).await.unwrap()
14554            );
14555            assert_eq!(
14556                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14557                "required commit"
14558            );
14559        }
14560        assert!(runner.state.worktree_root().join("judge-1").exists());
14561        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14562        runner.prep().await.unwrap();
14563        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14564
14565        runner.state.prep_complete = Some(false);
14566        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14567            run: Some(format!(
14568                "test -f '{}'",
14569                tmp.path().join("dependency-ready").display()
14570            )),
14571            ..Default::default()
14572        }];
14573        assert!(runner.prep().await.is_err());
14574        assert!(runner.state.prep_failure.is_some());
14575        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14576        runner.state = RunState::load(&runner.state.id).unwrap();
14577        assert!(
14578            runner.prep().await.is_err(),
14579            "correcting the cause alone must not auto-retry"
14580        );
14581        let failed_at = runner.state.events.last().unwrap().at;
14582        assert!(
14583            runner.retry_preparation(failed_at).is_err(),
14584            "old approvals cannot unlock the hold"
14585        );
14586        let mut task = crate::queue::Task::new(
14587            "retry".into(),
14588            "retry setup".into(),
14589            repo.clone(),
14590            crate::queue::Source::Human,
14591        );
14592        task.start(runner.state.id.clone());
14593        task.hold_machine(Some("setup failed".into()));
14594        task.release_operator("cli", None);
14595        runner
14596            .retry_preparation(task.resumption.as_ref().unwrap().at)
14597            .unwrap();
14598        runner.prep().await.unwrap();
14599        assert!(runner.state.prep_failure.is_none());
14600        assert_eq!(runner.state.prep_complete, Some(true));
14601        assert_eq!(runner.state.candidates.len(), 2);
14602        assert!(runner.state.worktree_root().join("judge-1").exists());
14603        for candidate in &runner.state.candidates {
14604            assert_eq!(
14605                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14606                git::tree_of(&repo, &seed).await.unwrap()
14607            );
14608        }
14609    }
14610
14611    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14612        let mut heads = Vec::new();
14613        for candidate in candidates {
14614            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14615        }
14616        heads
14617    }
14618
14619    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14620        ask_test_home();
14621        let tmp = tempfile::tempdir().unwrap();
14622        let repo = tmp.path().join("repo");
14623        std::fs::create_dir(&repo).unwrap();
14624        init_repo(&repo);
14625        let mut runner = runner_at(RunStatus::Reviewing);
14626        runner.state.repo = repo.clone();
14627        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14628        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14629        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14630            let branch = format!("candidate-{label}");
14631            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14632            git::worktree_add_branch(&repo, &worktree, &branch, "main")
14633                .await
14634                .unwrap();
14635            if label == 'B' {
14636                std::fs::write(worktree.join("change"), "winner").unwrap();
14637                git::rescue_commit(&worktree, "candidate change")
14638                    .await
14639                    .unwrap();
14640            }
14641            runner.state.candidates.push(Candidate {
14642                index,
14643                label,
14644                agent: "alpha".into(),
14645                branch,
14646                worktree,
14647                summary: "completed reply".into(),
14648                stat: String::new(),
14649                files: usize::from(label == 'B'),
14650                commits: usize::from(label == 'B'),
14651                empty: label == 'A',
14652                failed: None,
14653                verified_noop: None,
14654                duration_ms: 1,
14655                folded: false,
14656            });
14657        }
14658        runner.state.judge_skipped = true;
14659        runner.state.tally = Some(Tally {
14660            first_choice: BTreeMap::new(),
14661            borda: BTreeMap::new(),
14662            winner: 'B',
14663            rankings: 0,
14664            unanimous_initial: false,
14665            deliberated: false,
14666            changed_votes: 0,
14667            unanimous_final: false,
14668            tie_break: None,
14669            judges: 0,
14670            present: 0,
14671            quorum: 0,
14672            met_quorum: true,
14673            uncontested: Some("only B produced a change".into()),
14674        });
14675        runner.state.save().unwrap();
14676        (tmp, runner)
14677    }
14678
14679    #[tokio::test]
14680    #[cfg(unix)]
14681    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14682        for (solo, failed_checkpoint) in
14683            [(true, false), (false, false), (true, true), (false, true)]
14684        {
14685            let (_tmp, mut runner) = preservation_resume_fixture().await;
14686            if solo {
14687                runner.state.candidates.retain(|c| c.label == 'A');
14688            }
14689            runner.state.candidates[0].empty = false;
14690            runner.state.candidates[0].summary.clear();
14691            runner.state.tally = None;
14692            runner.state.judge_skipped = false;
14693            runner.state.status = RunStatus::Implementing;
14694            runner.state.config.disk.min_free_bytes = u64::MAX;
14695            let mut implementer = spec("alpha");
14696            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14697            runner.roles.implementers = vec![implementer];
14698            let patch = runner.state.dir().join("artifacts/cand-A.patch");
14699            if failed_checkpoint {
14700                std::fs::create_dir_all(&patch).unwrap();
14701            }
14702            runner.state.save().unwrap();
14703            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14704            runner.state = RunState::load(&runner.state.id).unwrap();
14705            assert_eq!(runner.state.status, RunStatus::Blocked);
14706            assert!(runner.state.interrupted_implementations.contains(&'A'));
14707            assert_eq!(runner.state.candidates[0].commits, 0);
14708            assert!(!runner.state.candidates[0].empty);
14709            assert!(runner.state.candidates[0].failed.is_none());
14710            if failed_checkpoint {
14711                assert!(runner.state.candidate_preservation.contains_key(&'A'));
14712                std::fs::remove_dir(patch).unwrap();
14713            }
14714            runner.state.resource_block = None;
14715            runner.state.config.disk.min_free_bytes = 0;
14716            runner.roles.implementers[0].command = vec![
14717                "sh".into(),
14718                "-c".into(),
14719                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14720            ];
14721            runner.implement().await.unwrap();
14722            assert!(runner.state.interrupted_implementations.is_empty());
14723            assert!(runner.state.candidate_preservation.is_empty());
14724            assert_eq!(runner.state.candidates[0].commits, 1);
14725            assert!(!runner.state.candidates[0].empty);
14726            assert!(
14727                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14728                    .unwrap()
14729                    .contains("retried empty interrupted seat")
14730            );
14731            if !solo {
14732                assert_eq!(runner.state.candidates[1].commits, 1);
14733            }
14734        }
14735    }
14736
14737    #[tokio::test]
14738    #[cfg(unix)]
14739    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14740        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14741            let (_tmp, mut runner) = preservation_resume_fixture().await;
14742            runner.state.candidates.retain(|c| c.label == 'A');
14743            runner.state.candidates[0].empty = false;
14744            runner.state.candidates[0].summary.clear();
14745            runner.state.tally = None;
14746            runner.state.judge_skipped = false;
14747            runner.state.status = RunStatus::Implementing;
14748            // Force the real post-spawn watchdog, independently of available
14749            // disk space. Dirty/committed work predates its first measurement.
14750            runner.state.config.disk.min_free_bytes = u64::MAX;
14751            let wt = runner.state.candidates[0].worktree.clone();
14752            std::fs::write(
14753                wt.join("interrupted-source"),
14754                "retain implementation work\n",
14755            )
14756            .unwrap();
14757            if committed {
14758                git::rescue_commit(&wt, "implementation checkpoint")
14759                    .await
14760                    .unwrap();
14761            }
14762            let lock = git::git(
14763                &wt,
14764                &[
14765                    "rev-parse",
14766                    "--path-format=absolute",
14767                    "--git-path",
14768                    "index.lock",
14769                ],
14770            )
14771            .await
14772            .unwrap();
14773            if block_rescue {
14774                std::fs::write(&lock, "fixture-owned lock").unwrap();
14775            }
14776            let mut implementer = spec("alpha");
14777            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14778            runner.roles.implementers = vec![implementer];
14779            runner.state.save().unwrap();
14780            let result = runner.implement().await;
14781            assert_eq!(result.is_err(), block_rescue);
14782            assert_eq!(
14783                runner.state.resource_block.as_ref().unwrap().kind,
14784                "disk pressure"
14785            );
14786            assert!(
14787                runner
14788                    .state
14789                    .events
14790                    .iter()
14791                    .any(|e| e.message.contains("agent execution interrupted"))
14792            );
14793            assert!(
14794                runner
14795                    .state
14796                    .dir()
14797                    .join("artifacts/candidate-A-recovery/staged.patch")
14798                    .exists()
14799            );
14800            if block_rescue {
14801                let saved = RunState::load(&runner.state.id).unwrap();
14802                assert!(saved.candidate_preservation.contains_key(&'A'));
14803                assert!(!saved.candidates[0].empty);
14804                assert_eq!(
14805                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14806                    "retain implementation work\n"
14807                );
14808                std::fs::remove_file(lock).unwrap();
14809                runner.state = saved;
14810            } else {
14811                assert_eq!(runner.state.status, RunStatus::Blocked);
14812                assert_eq!(runner.state.candidates[0].commits, 1);
14813            }
14814            // Recovery must save the candidate before deciding whether another
14815            // implementation is needed. No implementer is available on retry.
14816            runner.roles.implementers.clear();
14817            runner.state.config.disk.min_free_bytes = 0;
14818            runner.state.resource_block = None;
14819            runner.implement().await.unwrap();
14820            assert!(runner.state.candidate_preservation.is_empty());
14821            assert_eq!(runner.state.candidates[0].commits, 1);
14822            assert!(!runner.state.candidates[0].empty);
14823            assert!(git::is_clean(&wt).await.unwrap());
14824            assert!(
14825                crate::run::read_artifact(&runner.state, "cand-A.patch")
14826                    .unwrap()
14827                    .contains("retain implementation work")
14828            );
14829        }
14830    }
14831
14832    #[tokio::test]
14833    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14834        let (_tmp, mut runner) = preservation_resume_fixture().await;
14835        // Schema 15 already confirmed the unfolded no-change result.
14836        runner.implement().await.unwrap();
14837        assert!(runner.state.candidate_preservation.is_empty());
14838        assert!(
14839            !runner
14840                .state
14841                .dir()
14842                .join("artifacts/candidate-A-recovery")
14843                .exists()
14844        );
14845        runner.fold_losers().await.unwrap();
14846        assert!(!runner.state.candidates[0].worktree.exists());
14847        for schema in [14, crate::run::SCHEMA] {
14848            runner.state.schema = schema;
14849            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14850            runner.state.save().unwrap();
14851            runner.state = RunState::load(&runner.state.id).unwrap();
14852            runner.implement().await.unwrap();
14853            assert!(runner.state.candidate_preservation.is_empty());
14854            assert!(runner.state.candidates[0].folded);
14855            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14856        }
14857    }
14858
14859    #[tokio::test]
14860    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
14861        let (_tmp, mut runner) = preservation_resume_fixture().await;
14862        runner.state.candidates.retain(|c| c.label == 'A');
14863        runner.state.tally = None;
14864        runner.state.judge_skipped = false;
14865        let candidate = &mut runner.state.candidates[0];
14866        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
14867        candidate.failed = Some("deployment validation unavailable".into());
14868        std::fs::write(
14869            candidate.worktree.join("repair"),
14870            "future launcher policy\n",
14871        )
14872        .unwrap();
14873        runner
14874            .state
14875            .candidate_preservation
14876            .insert('A', "awaiting persistence".into());
14877        runner.persist_candidates().await.unwrap();
14878        let candidate = &runner.state.candidates[0];
14879        assert!(!candidate.empty);
14880        assert_eq!(candidate.commits, 1);
14881        assert_eq!(candidate.files, 1);
14882        assert!(candidate.failed.is_none());
14883        assert!(candidate.summary.contains("EPERM"));
14884        assert!(
14885            crate::run::read_artifact(&runner.state, "cand-A.patch")
14886                .unwrap()
14887                .contains("future launcher policy")
14888        );
14889        assert!(runner.state.candidate_preservation.is_empty());
14890    }
14891
14892    #[tokio::test]
14893    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14894        let (_tmp, mut runner) = preservation_resume_fixture().await;
14895        let wt = runner.state.candidates[0].worktree.clone();
14896        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14897        runner.state.schema = 14;
14898        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14899        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14900        let tally = serde_json::to_value(&runner.state.tally).unwrap();
14901        for _ in 0..2 {
14902            assert!(runner.implement().await.is_err());
14903            assert_eq!(runner.state.status, RunStatus::Blocked);
14904            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14905            assert!(
14906                runner.state.candidates[0].empty,
14907                "old verdict must remain unusable"
14908            );
14909            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14910            assert!(runner.state.judge_skipped);
14911            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14912            assert!(
14913                crate::run::read_artifact(&runner.state, "cand-A.patch")
14914                    .unwrap()
14915                    .contains("owned edits")
14916            );
14917            assert!(wt.exists());
14918            assert_eq!(
14919                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14920                    .await
14921                    .unwrap(),
14922                1
14923            );
14924            runner.state = RunState::load(&runner.state.id).unwrap();
14925        }
14926    }
14927
14928    #[tokio::test]
14929    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14930        let (_tmp, mut runner) = preservation_resume_fixture().await;
14931        let wt = runner.state.candidates[0].worktree.clone();
14932        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14933        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14934        let index_lock = PathBuf::from(
14935            git::git(
14936                &wt,
14937                &[
14938                    "rev-parse",
14939                    "--path-format=absolute",
14940                    "--git-path",
14941                    "index.lock",
14942                ],
14943            )
14944            .await
14945            .unwrap(),
14946        );
14947        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14948        runner.state.schema = 14;
14949        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14950        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14951        assert!(runner.implement().await.is_err());
14952        assert!(runner.after_implement().is_err());
14953        assert!(runner.fold_losers().await.is_err());
14954        assert_eq!(
14955            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14956            "failed seat edits"
14957        );
14958        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14959        runner.state = RunState::load(&runner.state.id).unwrap();
14960        std::fs::remove_file(index_lock).unwrap();
14961        // Recovery finds changes that the old uncontested result never judged.
14962        assert!(runner.implement().await.is_err());
14963        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14964        assert!(runner.state.candidates[0].empty);
14965        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14966        assert!(
14967            crate::run::read_artifact(&runner.state, "cand-A.patch")
14968                .unwrap()
14969                .contains("failed seat edits")
14970        );
14971        assert!(wt.exists());
14972    }
14973
14974    #[tokio::test]
14975    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14976        let (_tmp, mut runner) = preservation_resume_fixture().await;
14977        let repo = runner.state.repo.clone();
14978        let common = git::git(
14979            &repo,
14980            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14981        )
14982        .await
14983        .unwrap();
14984        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14985        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14986        assert!(runner.fold_losers().await.is_err());
14987        assert!(!runner.state.candidates[0].worktree.exists());
14988        assert!(!runner.state.candidates[0].folded);
14989        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14990        runner.state = RunState::load(&runner.state.id).unwrap();
14991        std::fs::remove_file(branch_lock).unwrap();
14992        let judge = runner.state.worktree_root().join("judge-1");
14993        git::worktree_add_detached(&repo, &judge, "main")
14994            .await
14995            .unwrap();
14996        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14997        runner.roles.judges = vec![spec("judge")];
14998        assert!(runner.fold_losers().await.is_err());
14999        runner.state = RunState::load(&runner.state.id).unwrap();
15000        assert!(
15001            runner.state.candidates[0].folded,
15002            "candidate progress saved before judge failure"
15003        );
15004        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
15005        assert!(judge.join("uncommitted").exists());
15006        std::fs::remove_file(judge.join("uncommitted")).unwrap();
15007        runner.fold_losers().await.unwrap();
15008        assert!(!judge.exists());
15009        runner.fold_losers().await.unwrap();
15010    }
15011
15012    #[tokio::test]
15013    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
15014        let (_tmp, mut runner) = preservation_resume_fixture().await;
15015        runner.state.candidates.retain(|c| c.label == 'B');
15016        runner.state.tally = None;
15017        runner.state.judge_skipped = false;
15018        runner.state.prep_complete = Some(true);
15019        runner.state.config.graph.advise = false;
15020        runner.state.config.graph.selected_implementer = Some("alpha".into());
15021        let wt = runner.state.candidates[0].worktree.clone();
15022        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
15023        runner
15024            .state
15025            .candidate_preservation
15026            .insert('B', "awaiting persistence".into());
15027        let index_lock = git::git(
15028            &wt,
15029            &[
15030                "rev-parse",
15031                "--path-format=absolute",
15032                "--git-path",
15033                "index.lock",
15034            ],
15035        )
15036        .await
15037        .unwrap();
15038        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15039        runner.state.save().unwrap();
15040        assert!(runner.execute_graph().await.is_err());
15041        assert_eq!(runner.state.status, RunStatus::Blocked);
15042        assert!(runner.state.candidate_preservation.contains_key(&'B'));
15043        std::fs::remove_file(index_lock).unwrap();
15044        // Park at the next graph boundary: this tests the real selected-seat
15045        // hold guard without dispatching reviewers or publishing anything.
15046        for _ in 0..2 {
15047            runner.state = RunState::load(&runner.state.id).unwrap();
15048            runner.pause.park();
15049            runner.execute_graph().await.unwrap();
15050            assert_eq!(runner.state.status, RunStatus::Implementing);
15051            assert!(runner.state.candidate_preservation.is_empty());
15052            assert_eq!(runner.state.candidates[0].commits, 2);
15053            assert!(runner.state.tally.is_none());
15054        }
15055        runner.implement().await.unwrap();
15056        assert_eq!(runner.state.status, RunStatus::Judging);
15057        // A genuine selected-implementer hold must still require release.
15058        runner.state.status = RunStatus::Blocked;
15059        runner.state.candidates[0].failed = Some("implementation failed".into());
15060        runner.state.save().unwrap();
15061        let error = runner.execute_graph().await.unwrap_err();
15062        assert!(
15063            error
15064                .to_string()
15065                .contains("selected implementation is held")
15066        );
15067    }
15068
15069    #[tokio::test]
15070    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
15071        let (_tmp, mut runner) = preservation_resume_fixture().await;
15072        let repo = runner.state.repo.clone();
15073        let candidate = runner.state.candidates[0].clone();
15074        // Remove only the disposable fixture directory, leaving Git's linked
15075        // worktree registration and branch intact, as in a vanished checkout.
15076        let registered = git::worktree_holding(&repo, &candidate.branch)
15077            .await
15078            .unwrap()
15079            .unwrap();
15080        assert_eq!(
15081            registered.canonicalize().unwrap(),
15082            candidate.worktree.canonicalize().unwrap()
15083        );
15084        std::fs::remove_dir_all(&candidate.worktree).unwrap();
15085        assert_eq!(
15086            git::worktree_holding(&repo, &candidate.branch)
15087                .await
15088                .unwrap(),
15089            Some(registered)
15090        );
15091        assert!(
15092            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
15093            "the stale registration initially keeps the branch checked out"
15094        );
15095        runner.fold_losers().await.unwrap();
15096        assert!(runner.state.candidates[0].folded);
15097        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15098        assert_eq!(
15099            git::worktree_holding(&repo, &candidate.branch)
15100                .await
15101                .unwrap(),
15102            None
15103        );
15104        runner.state = RunState::load(&runner.state.id).unwrap();
15105        runner.fold_losers().await.unwrap();
15106    }
15107
15108    #[tokio::test]
15109    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
15110        let (_tmp, mut runner) = preservation_resume_fixture().await;
15111        let repo = runner.state.repo.clone();
15112        let candidate = runner.state.candidates[0].clone();
15113        let winner = runner.state.candidates[1].clone();
15114        let registered = git::worktree_holding(&repo, &candidate.branch)
15115            .await
15116            .unwrap()
15117            .unwrap();
15118        assert_eq!(
15119            registered.canonicalize().unwrap(),
15120            candidate.worktree.canonicalize().unwrap()
15121        );
15122        std::fs::remove_dir_all(&candidate.worktree).unwrap();
15123        assert_eq!(
15124            git::worktree_holding(&repo, &candidate.branch)
15125                .await
15126                .unwrap(),
15127            Some(registered)
15128        );
15129        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
15130
15131        for _ in 0..2 {
15132            fold_run(&mut runner.state, false, &crate::run::home())
15133                .await
15134                .unwrap();
15135            runner.state = RunState::load(&runner.state.id).unwrap();
15136            assert!(runner.state.candidates[0].folded);
15137            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15138            assert_eq!(
15139                git::worktree_holding(&repo, &candidate.branch)
15140                    .await
15141                    .unwrap(),
15142                None
15143            );
15144            assert!(!runner.state.candidates[1].folded);
15145            assert!(winner.worktree.exists());
15146            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
15147        }
15148    }
15149
15150    #[tokio::test]
15151    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
15152        ask_test_home();
15153        let tmp = tempfile::tempdir().unwrap();
15154        let repo = tmp.path().join("repo");
15155        std::fs::create_dir(&repo).unwrap();
15156        init_repo(&repo);
15157        let mut runner = runner_at(RunStatus::Implementing);
15158        runner.state.repo = repo.clone();
15159        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15160        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15161        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15162            let branch = format!("candidate-{label}");
15163            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
15164            git::worktree_add_branch(&repo, &wt, &branch, "main")
15165                .await
15166                .unwrap();
15167            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
15168            git::git(&wt, &["add", "README.md"]).await.unwrap();
15169            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
15170            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
15171            runner.state.candidates.push(Candidate {
15172                index,
15173                label,
15174                agent: "alpha".to_owned(),
15175                branch,
15176                worktree: wt,
15177                summary: "completed implementation".to_owned(),
15178                stat: String::new(),
15179                files: 0,
15180                commits: 0,
15181                empty: false,
15182                failed: None,
15183                verified_noop: None,
15184                duration_ms: 42,
15185                folded: false,
15186            });
15187            runner
15188                .state
15189                .candidate_preservation
15190                .insert(label, "awaiting persistence".to_owned());
15191        }
15192        let wt = runner.state.candidates[0].worktree.clone();
15193        let index_lock = git::git(
15194            &wt,
15195            &[
15196                "rev-parse",
15197                "--path-format=absolute",
15198                "--git-path",
15199                "index.lock",
15200            ],
15201        )
15202        .await
15203        .unwrap();
15204        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15205        runner.state.save().unwrap();
15206        assert!(runner.persist_candidates().await.is_err());
15207        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15208        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
15209        assert_eq!(
15210            runner.state.candidates[1].commits, 1,
15211            "other results are still collected"
15212        );
15213        assert!(
15214            !runner.state.candidates[0].empty,
15215            "failed rescue is not no change"
15216        );
15217        assert_eq!(
15218            runner.state.candidates[0].summary,
15219            "completed implementation"
15220        );
15221        assert!(runner.after_implement().is_err());
15222        let home = tmp.path().join("fold-home");
15223        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
15224        assert!(wt.join("new-file").exists());
15225        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
15226        assert!(recovery.join("staged.patch").exists());
15227        assert!(recovery.join("unstaged.patch").exists());
15228        assert_eq!(
15229            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
15230            "untracked edits\n"
15231        );
15232        runner.state.begin_pass();
15233        runner.state.end_pass_parked(ParkCause::Upgrade {
15234            from: "1.0".into(),
15235            to: Some("1.1".into()),
15236        });
15237        runner.state.save().unwrap();
15238        runner.state = RunState::load(&runner.state.id).unwrap();
15239        assert!(matches!(
15240            runner.state.passes.last().unwrap().end,
15241            Some(crate::run::PassEnd::Parked {
15242                cause: ParkCause::Upgrade { .. }
15243            })
15244        ));
15245        std::fs::remove_file(index_lock).unwrap();
15246        // No implementer is configured: a persistence retry cannot dispatch a seat.
15247        runner.implement().await.unwrap();
15248        assert!(runner.state.candidate_preservation.is_empty());
15249        assert!(
15250            runner
15251                .state
15252                .candidates
15253                .iter()
15254                .all(|c| c.commits == 1 && !c.empty)
15255        );
15256        assert!(git::is_clean(&wt).await.unwrap());
15257        assert!(
15258            crate::run::read_artifact(&runner.state, "cand-A.patch")
15259                .unwrap()
15260                .contains("later tracked edits")
15261        );
15262    }
15263
15264    #[tokio::test]
15265    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
15266        ask_test_home();
15267        let tmp = tempfile::tempdir().unwrap();
15268        let repo = tmp.path().join("repo");
15269        std::fs::create_dir(&repo).unwrap();
15270        init_repo(&repo);
15271        let wt = tmp.path().join("candidate");
15272        git::worktree_add_branch(&repo, &wt, "candidate", "main")
15273            .await
15274            .unwrap();
15275        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15276        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
15277        assert!(wt.join("uncommitted").exists());
15278        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15279        std::fs::remove_file(wt.join("uncommitted")).unwrap();
15280        let mut runner = runner_at(RunStatus::Implementing);
15281        runner.state.repo = repo.clone();
15282        runner.state.base_commit = "missing-base".to_owned();
15283        runner.state.candidates.push(Candidate {
15284            index: 0,
15285            label: 'A',
15286            agent: "alpha".into(),
15287            branch: "candidate".into(),
15288            worktree: wt.clone(),
15289            summary: "completed".into(),
15290            stat: String::new(),
15291            files: 0,
15292            commits: 0,
15293            empty: false,
15294            failed: None,
15295            verified_noop: None,
15296            duration_ms: 1,
15297            folded: false,
15298        });
15299        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15300        assert!(
15301            fold_run(&mut runner.state, true, &tmp.path().join("home"))
15302                .await
15303                .is_err()
15304        );
15305        assert!(!runner.state.candidates[0].folded);
15306        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15307        std::fs::remove_file(wt.join("uncommitted")).unwrap();
15308        runner
15309            .state
15310            .candidate_preservation
15311            .insert('A', "pending".into());
15312        assert!(runner.persist_candidates().await.is_err());
15313        assert!(!runner.state.candidates[0].empty);
15314        assert!(!runner.state.candidates[0].folded);
15315        assert!(wt.exists());
15316        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15317        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
15318        std::fs::create_dir_all(&patch_path).unwrap();
15319        assert!(
15320            runner.persist_candidates().await.is_err(),
15321            "artifact failure stays pending"
15322        );
15323        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15324        assert!(!runner.state.candidates[0].empty);
15325        std::fs::remove_dir(patch_path).unwrap();
15326        runner.persist_candidates().await.unwrap();
15327        assert!(
15328            runner.state.candidates[0].empty,
15329            "only successful inspection confirms no change"
15330        );
15331    }
15332
15333    fn init_repo(dir: &Path) {
15334        let run = |args: &[&str]| {
15335            let out = std::process::Command::new("git")
15336                .args(args)
15337                .current_dir(dir)
15338                .quiet()
15339                .output()
15340                .expect("spawn git");
15341            assert!(
15342                out.status.success(),
15343                "git {args:?} failed: {}",
15344                String::from_utf8_lossy(&out.stderr)
15345            );
15346        };
15347        run(&["init", "-b", "main"]);
15348        run(&["config", "user.name", "magi test"]);
15349        run(&["config", "user.email", "magi@example.com"]);
15350        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
15351        run(&["add", "-A"]);
15352        run(&["commit", "-m", "init"]);
15353    }
15354
15355    // `settle_questions` is what closes the ghost the phone showed: a run's
15356    // seat asked something, the run then ended, and nothing was left to
15357    // abandon the question it left `open`. `HOME` is a process-wide
15358    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
15359    // first time it runs in the binary — every test below still reaches the
15360    // same directory whichever call won, and each gets its own run id from
15361    // `RunState::new`, so they never collide there.
15362    fn ask_test_home() {
15363        crate::run::pin_test_home();
15364    }
15365
15366    #[cfg(unix)]
15367    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
15368        ask_test_home();
15369        let tmp = tempfile::tempdir().unwrap();
15370        let repo = tmp.path().join("repo");
15371        std::fs::create_dir(&repo).unwrap();
15372        init_repo(&repo);
15373        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15374        let origin = tmp.path().join("origin.git");
15375        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
15376            .await
15377            .unwrap();
15378        git::git(
15379            &repo,
15380            &["remote", "add", "origin", origin.to_str().unwrap()],
15381        )
15382        .await
15383        .unwrap();
15384        git::git(&repo, &["checkout", "-b", "fix/retained"])
15385            .await
15386            .unwrap();
15387        std::fs::write(repo.join("retained"), "retained change").unwrap();
15388        git::git(&repo, &["add", "."]).await.unwrap();
15389        git::git(&repo, &["commit", "-m", "retained implementation"])
15390            .await
15391            .unwrap();
15392        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15393        git::git(&repo, &["checkout", "main"]).await.unwrap();
15394        std::fs::write(repo.join("base-change"), "new base").unwrap();
15395        git::git(&repo, &["add", "."]).await.unwrap();
15396        git::git(&repo, &["commit", "-m", "base moved"])
15397            .await
15398            .unwrap();
15399        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15400        git::git(&repo, &["checkout", "fix/retained"])
15401            .await
15402            .unwrap();
15403        let mut runner = runner_at(RunStatus::Voting);
15404        runner.state.repo = repo.clone();
15405        runner.state.base_commit = base;
15406        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15407        runner.state.config.disk.min_free_bytes = 0;
15408        runner.state.config.graph.review_rounds = 1;
15409        runner.state.config.verify.e2e = vec!["true".into()];
15410        runner.state.config.verify.gate = vec!["true".into()];
15411        runner.state.candidates = vec![Candidate {
15412            index: 0,
15413            label: 'A',
15414            agent: "fixture".into(),
15415            branch: "fix/retained".into(),
15416            worktree: repo.clone(),
15417            summary: String::new(),
15418            stat: String::new(),
15419            files: 1,
15420            commits: 1,
15421            empty: false,
15422            failed: None,
15423            verified_noop: None,
15424            duration_ms: 0,
15425            folded: false,
15426        }];
15427        runner.state.tally = Some(Tally {
15428            first_choice: BTreeMap::new(),
15429            borda: BTreeMap::new(),
15430            winner: 'A',
15431            rankings: 0,
15432            unanimous_initial: true,
15433            deliberated: false,
15434            changed_votes: 0,
15435            unanimous_final: true,
15436            tie_break: None,
15437            judges: 0,
15438            present: 0,
15439            quorum: 0,
15440            met_quorum: true,
15441            uncontested: Some("review-only".into()),
15442        });
15443        let mut task = crate::queue::Task::new(
15444            "fixture".into(),
15445            "fixture".into(),
15446            repo,
15447            crate::queue::Source::Human,
15448        );
15449        task.link_run(&runner.state.id);
15450        crate::queue::Queue::open().put(&mut task).unwrap();
15451        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
15452        runner.state.base_sync = Some(BaseSync {
15453            tip: String::new(),
15454            behind: 1,
15455            attempts: BASE_SYNC_ROUNDS,
15456            conflict: None,
15457            already_in: None,
15458        });
15459        runner.sync_to_base().await.unwrap();
15460        assert_eq!(runner.state.status, RunStatus::Blocked);
15461        (tmp, runner, head, task.id)
15462    }
15463
15464    #[cfg(unix)]
15465    #[tokio::test]
15466    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
15467        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
15468        if std::env::var_os(CHILD).is_none() {
15469            let home = tempfile::tempdir().unwrap();
15470            let output = std::process::Command::new(std::env::current_exe().unwrap())
15471                .args([
15472                    "--exact",
15473                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
15474                    "--nocapture",
15475                ])
15476                .env(CHILD, "1")
15477                .env("MAGI_HOME", home.path())
15478                .env("MAGI_WORKER_HOME", home.path())
15479                .env("MAGI_WORKER_ID", "abcdef")
15480                .output()
15481                .unwrap();
15482            assert!(
15483                output.status.success()
15484                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15485                "{}\n{}",
15486                String::from_utf8_lossy(&output.stdout),
15487                String::from_utf8_lossy(&output.stderr)
15488            );
15489            return;
15490        }
15491        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
15492        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15493        let home = crate::run::home();
15494        let queue = crate::queue::Queue::open();
15495        let mut owner = queue.get(&task).unwrap();
15496        owner.release_operator("test-release", None);
15497        owner.start(runner.state.id.clone());
15498        queue.put(&mut owner).unwrap();
15499        runner.state.driver_exited = true;
15500        runner.state.task_instruction = Some(owner.instruction.clone());
15501        runner.state.save().unwrap();
15502        let _claim = queue.claim(&task).unwrap();
15503        let reserve = |id: &str| {
15504            let directory = home.join("workers/jobs").join(id);
15505            std::fs::create_dir_all(&directory).unwrap();
15506            let job = crate::worker::Job {
15507                id: id.into(),
15508                sequence: 1,
15509                kind: "task".into(),
15510                key: format!("task:{task}"),
15511                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
15512                generation: "fixture-generation".into(),
15513                version: None,
15514                status: crate::worker::Status::Running,
15515                result: None,
15516                pid: Some(std::process::id()),
15517                created_at: 0,
15518                finished_at: None,
15519            };
15520            std::fs::write(
15521                directory.join("job.json"),
15522                serde_json::to_vec(&job).unwrap(),
15523            )
15524            .unwrap();
15525            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
15526        };
15527        let _own = reserve("abcdef");
15528        let other = reserve("fedcba");
15529        assert!(
15530            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15531                .await
15532                .is_err()
15533        );
15534        assert!(
15535            RunState::load(&runner.state.id)
15536                .unwrap()
15537                .base_sync_recovery
15538                .authorizations
15539                .is_empty()
15540        );
15541        drop(other);
15542        assert!(crate::daemon::is_working_on(
15543            &home,
15544            &runner.state.id,
15545            Timestamp::now()
15546        ));
15547        assert!(
15548            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
15549        );
15550        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15551            .await
15552            .unwrap();
15553        assert!(
15554            RunState::load(&runner.state.id)
15555                .unwrap()
15556                .base_recovery_pending()
15557        );
15558    }
15559
15560    #[cfg(unix)]
15561    #[tokio::test]
15562    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
15563        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15564        let home = crate::run::home();
15565        runner.state.driver_pid = Some(std::process::id());
15566        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
15567        runner.state.driver_exited = false;
15568        runner.state.begin_pass();
15569        runner.state.save().unwrap();
15570        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
15571        driver.record(&runner.state).unwrap();
15572        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
15573        // Preserve the durable claim as if the process died before Drop saved.
15574        drop(driver);
15575        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
15576        assert_eq!(
15577            runner.state.liveness_under(&home, false),
15578            crate::run::Liveness::Dead
15579        );
15580        let queue = crate::queue::Queue::open();
15581        let _claim = queue.claim(&task).unwrap();
15582        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15583            .await
15584            .unwrap();
15585        let disk = RunState::load(&runner.state.id).unwrap();
15586        assert!(!disk.driver_exited);
15587        assert!(disk.base_recovery_pending());
15588    }
15589
15590    #[cfg(unix)]
15591    #[tokio::test]
15592    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
15593        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
15594        if std::env::var_os(CHILD).is_none() {
15595            use std::os::unix::fs::PermissionsExt;
15596            let tmp = tempfile::tempdir().unwrap();
15597            let gh = tmp.path().join("gh");
15598            std::fs::write(&gh, r#"#!/bin/sh
15599case "$1 $2" in
15600  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
15601  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
15602  *) exit 2 ;;
15603esac
15604"#).unwrap();
15605            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
15606            let mut paths = vec![tmp.path().to_path_buf()];
15607            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
15608            let output = std::process::Command::new(std::env::current_exe().unwrap())
15609                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
15610                .env(CHILD, "1")
15611                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
15612                .env("PATH", std::env::join_paths(paths).unwrap())
15613                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
15614                .output().unwrap();
15615            assert!(
15616                output.status.success()
15617                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15618                "{}\n{}",
15619                String::from_utf8_lossy(&output.stdout),
15620                String::from_utf8_lossy(&output.stderr)
15621            );
15622            return;
15623        }
15624        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15625        let repo = runner.state.repo.clone();
15626        git::git(&repo, &["checkout", "main"]).await.unwrap();
15627        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
15628        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15629        git::git(&repo, &["checkout", "fix/retained"])
15630            .await
15631            .unwrap();
15632        runner.state.pr = Some(crate::run::PrRecord {
15633            url: "https://example.test/project/pull/1".into(),
15634            number: 1,
15635            state: "open".into(),
15636            checks: "unknown".into(),
15637            round: 0,
15638            rounds: 1,
15639            red_at_merge: Vec::new(),
15640        });
15641        runner.sync_to_base().await.unwrap();
15642        assert_eq!(runner.state.status, RunStatus::Blocked);
15643        assert_eq!(
15644            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
15645            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
15646        );
15647        assert!(runner.state.base_sync_closure_pending());
15648        assert!(crate::handover::can_resume(&runner.state));
15649        let queue = crate::queue::Queue::open();
15650        assert!(!crate::base_sync_recovery::held_task(
15651            &queue,
15652            &queue.get(&task).unwrap()
15653        ));
15654        assert!(runner.state.authorize_base_sync(&task).is_err());
15655        // Legacy closure stops have no typed stop but must retain the checkpoint.
15656        runner.state.base_sync_recovery.stop = None;
15657        runner.state.status = RunStatus::Voting;
15658        runner.state.gate_ran = true;
15659        runner.state.config.graph.review_rounds = 0;
15660        runner.state.save().unwrap();
15661        assert!(crate::handover::can_resume(&runner.state));
15662        runner.state = RunState::load(&runner.state.id).unwrap();
15663        runner.execute_graph().await.unwrap();
15664        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
15665        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
15666        assert_eq!(
15667            runner.state.base_sync.as_ref().unwrap().attempts,
15668            BASE_SYNC_ROUNDS
15669        );
15670        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
15671        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15672        assert!(runner.state.reviews.is_empty());
15673        assert!(runner.state.gate.is_empty());
15674    }
15675
15676    #[cfg(unix)]
15677    #[tokio::test]
15678    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
15679        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15680        runner.state.driver_exited = true;
15681        runner.state.save().unwrap();
15682        let home = crate::run::home();
15683        let queue = crate::queue::Queue::open();
15684        let _claim = queue.claim(&task).unwrap();
15685        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
15686        assert!(
15687            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15688                .await
15689                .is_err()
15690        );
15691        drop(lease);
15692        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
15693        assert!(
15694            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15695                .await
15696                .is_err()
15697        );
15698        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
15699        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15700            .await
15701            .unwrap();
15702        let disk = RunState::load(&runner.state.id).unwrap();
15703        assert!(disk.base_recovery_pending());
15704        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
15705        assert!(
15706            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15707                .await
15708                .is_err()
15709        );
15710        assert_eq!(
15711            RunState::load(&runner.state.id)
15712                .unwrap()
15713                .base_sync_recovery
15714                .authorizations
15715                .len(),
15716            1
15717        );
15718        let mut owner = queue.get(&task).unwrap();
15719        owner.runs.push("20260101-000000-ffff".into());
15720        queue.put(&mut owner).unwrap();
15721        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15722            .await
15723            .unwrap_err();
15724        assert!(error.to_string().contains("run task ownership changed"));
15725    }
15726
15727    #[cfg(unix)]
15728    #[tokio::test]
15729    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
15730        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
15731        let mut legacy = serde_json::to_value(&runner.state).unwrap();
15732        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
15733        legacy["status"] = serde_json::json!("voting");
15734        runner.state = serde_json::from_value(legacy).unwrap();
15735        runner.state.save().unwrap();
15736        for _ in 0..3 {
15737            runner.execute_graph().await.unwrap();
15738            assert_eq!(runner.state.status, RunStatus::Blocked);
15739            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
15740            assert!(!crate::handover::can_resume(&runner.state));
15741            assert_eq!(
15742                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15743                head
15744            );
15745            assert!(runner.state.reviews.is_empty());
15746            assert!(!runner.state.gate_ran);
15747        }
15748        let notice = crate::notices::run_ended(&runner.state).unwrap();
15749        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
15750        runner.state.begin_pass();
15751        runner.state.status = RunStatus::Voting;
15752        runner.deliberate().await.unwrap();
15753        runner.vote().await.unwrap();
15754        runner.finish_after_tally().await.unwrap();
15755        assert_eq!(runner.state.status, RunStatus::Blocked);
15756        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
15757        assert_eq!(
15758            crate::notices::run_ended(&runner.state).unwrap().since,
15759            notice.since
15760        );
15761    }
15762
15763    #[cfg(unix)]
15764    #[tokio::test]
15765    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
15766        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15767        for detail in [
15768            "fix/retained conflicts with origin/main and did not rebase: conflict",
15769            "rebased fix/retained locally but origin refused the push",
15770            "unknown failure",
15771        ] {
15772            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
15773            runner.state.record_base_sync_stop(&head);
15774            assert!(runner.state.authorize_base_sync(&task).is_err());
15775            runner.state.status = RunStatus::Voting;
15776            runner.execute_graph().await.unwrap();
15777            assert_eq!(runner.state.status, RunStatus::Blocked);
15778            assert!(!crate::handover::can_resume(&runner.state));
15779            assert_eq!(
15780                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15781                head
15782            );
15783        }
15784    }
15785
15786    #[cfg(unix)]
15787    #[tokio::test]
15788    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
15789        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15790        runner.state.authorize_base_sync(&task).unwrap();
15791        assert!(runner.state.consume_base_recovery(&head));
15792        runner.state.save().unwrap();
15793        runner.state = RunState::load(&runner.state.id).unwrap();
15794        runner.execute_graph().await.unwrap();
15795        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15796        assert_eq!(runner.state.status, RunStatus::Blocked);
15797        assert!(!runner.state.consume_base_recovery(&head));
15798        assert!(!crate::handover::can_resume(&runner.state));
15799        runner.state.authorize_base_sync(&task).unwrap();
15800        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
15801        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15802        assert_eq!(
15803            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15804            head
15805        );
15806    }
15807
15808    #[cfg(unix)]
15809    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
15810        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
15811        // This is an ordinary gate stop, not an authorized base-budget recovery.
15812        runner.state.prep_complete = Some(true);
15813        runner.state.base_sync = None;
15814        runner.state.base_sync_recovery = Default::default();
15815        runner.state.config.graph.gate_fix_rounds = 1;
15816        let mut review = review_round(true, 0, 1, 1, false, true);
15817        review.head = head.clone();
15818        review.verified_head = Some(head.clone());
15819        runner.state.reviews = vec![review];
15820        runner.state.gate = vec![CommandOutcome {
15821            command: "old gate".into(),
15822            code: Some(1),
15823            output_tail: "old failure".into(),
15824            duration_ms: 0,
15825            resource_blocked: false,
15826        }];
15827        runner.state.gate_head = Some(head.clone());
15828        runner.state.gate_ran = true;
15829        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
15830            agent: "fixture".into(),
15831            failed: runner.state.gate.clone(),
15832            notes: "spent".into(),
15833            committed: false,
15834            error: None,
15835        }];
15836        runner.state.land_approval = Some(crate::run::LandApproval {
15837            question: "old approval".into(),
15838            head: head.clone(),
15839        });
15840        runner.state.status = RunStatus::Blocked;
15841        (tmp, runner, head)
15842    }
15843
15844    #[cfg(unix)]
15845    #[tokio::test]
15846    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
15847        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
15848            let (tmp, mut runner, old) = resumed_gate_fixture().await;
15849            let phases = tmp.path().join("phases");
15850            runner.state.config.verify.e2e = vec![format!(
15851                "echo e2e >> '{}'; {}",
15852                phases.display(),
15853                if e2e_ok {
15854                    "true"
15855                } else {
15856                    "echo new-e2e-failure; false"
15857                }
15858            )];
15859            runner.state.config.verify.gate = vec![format!(
15860                "echo gate >> '{}'; {}",
15861                phases.display(),
15862                if gate_ok {
15863                    "true"
15864                } else {
15865                    "echo new-gate-failure; false"
15866                }
15867            )];
15868            runner.state.save().unwrap();
15869            runner.state = RunState::load(&runner.state.id).unwrap();
15870            runner.sync_to_base().await.unwrap();
15871            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15872            assert_ne!(head, old);
15873            // Reentry after Git updated but before verification was invalidated.
15874            runner.state = RunState::load(&runner.state.id).unwrap();
15875            runner.execute_graph().await.unwrap();
15876            assert_eq!(runner.state.gate_fixes.len(), 1);
15877            assert_eq!(runner.state.reviews.len(), 1);
15878            assert_eq!(runner.state.reviews[0].head, old);
15879            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
15880            assert_eq!(runner.state.verification_history.len(), 1);
15881            let history = &runner.state.verification_history[0];
15882            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
15883            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
15884            assert!(history.archived_at.is_some());
15885            assert_eq!(history.fixes_spent, Some(1));
15886            assert_eq!(history.gate[0].output_tail, "old failure");
15887            let archive_events: Vec<_> = runner
15888                .state
15889                .events
15890                .iter()
15891                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
15892                .collect();
15893            assert_eq!(archive_events.len(), 1);
15894            assert!(archive_events[0].message.contains(&short(&old)));
15895            assert!(archive_events[0].message.contains(&short(&head)));
15896            assert!(
15897                archive_events[0]
15898                    .message
15899                    .contains("1 gate-fix round(s) already spent stay spent")
15900            );
15901            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
15902            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
15903            if e2e_ok {
15904                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
15905                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
15906            } else {
15907                assert!(!runner.state.gate_ran);
15908            }
15909            assert_eq!(
15910                runner.state.status,
15911                if e2e_ok && gate_ok {
15912                    RunStatus::Ready
15913                } else {
15914                    RunStatus::Blocked
15915                }
15916            );
15917            if !e2e_ok || !gate_ok {
15918                let cause = runner.state.verification_blocking_cause().unwrap();
15919                assert!(cause.contains(&head));
15920                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
15921                // A later successful progress event must not become the cause.
15922                runner.state.event("land", "rebased successfully");
15923                let notice = crate::notices::run_ended(&runner.state).unwrap();
15924                assert!(!format!("{notice:?}").contains("rebased successfully"));
15925                assert!(runner.state.merge.is_none());
15926            }
15927            let before = std::fs::read_to_string(&phases).unwrap();
15928            runner.state.save().unwrap();
15929            runner.state = RunState::load(&runner.state.id).unwrap();
15930            runner.review_loop().await.unwrap();
15931            runner.gate().await.unwrap();
15932            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
15933            assert_eq!(runner.state.verification_history.len(), 1);
15934            let history = &runner.state.verification_history[0];
15935            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
15936            assert!(history.archived_at.is_some());
15937            assert_eq!(history.fixes_spent, Some(1));
15938        }
15939    }
15940
15941    #[cfg(unix)]
15942    #[tokio::test]
15943    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
15944        let (tmp, mut runner, head) = resumed_gate_fixture().await;
15945        let marker = tmp.path().join("gate-ran");
15946        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
15947        runner.review_loop().await.unwrap();
15948        runner.gate().await.unwrap();
15949        assert_eq!(runner.state.status, RunStatus::Blocked);
15950        assert!(!marker.exists());
15951        assert!(runner.state.verification_history.is_empty());
15952        runner.state.gate_head = None;
15953        runner.gate().await.unwrap();
15954        assert!(marker.exists());
15955        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
15956        assert_eq!(runner.state.verification_history.len(), 1);
15957        assert_eq!(runner.state.gate_fixes.len(), 1);
15958    }
15959
15960    #[cfg(unix)]
15961    #[tokio::test]
15962    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
15963        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
15964        runner.sync_to_base().await.unwrap();
15965        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15966        runner.state.reconcile_verification_head(&head);
15967        // Both crash-before-command and resource wait are unresolved evidence.
15968        for outcomes in [
15969            Vec::new(),
15970            vec![CommandOutcome {
15971                command: "true".into(),
15972                code: None,
15973                output_tail: "cache unavailable".into(),
15974                duration_ms: 0,
15975                resource_blocked: true,
15976            }],
15977        ] {
15978            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
15979                head: head.clone(),
15980                passed: false,
15981                outcomes,
15982            });
15983            runner.state.gate_ran = false;
15984            runner.state.gate.clear();
15985            runner.state.save().unwrap();
15986            runner.state = RunState::load(&runner.state.id).unwrap();
15987            runner.merge().await.unwrap();
15988            assert!(runner.state.merge.is_none());
15989            runner.review_loop().await.unwrap();
15990            runner.gate().await.unwrap();
15991            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
15992            assert!(runner.state.gate_allows_progress());
15993            assert_eq!(runner.state.gate_fixes.len(), 1);
15994            assert!(runner.state.merge.is_none());
15995        }
15996        // Moving the winner after a successful gate cannot publish an unchecked SHA.
15997        git::git(
15998            &runner.state.repo,
15999            &["commit", "--allow-empty", "-m", "external update"],
16000        )
16001        .await
16002        .unwrap();
16003        runner.merge().await.unwrap();
16004        assert!(runner.state.merge.is_none());
16005        assert_eq!(runner.state.status, RunStatus::Blocked);
16006    }
16007
16008    #[cfg(unix)]
16009    #[tokio::test]
16010    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
16011        for waiting_on_gate in [false, true] {
16012            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
16013            runner.sync_to_base().await.unwrap();
16014            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16015            runner.state.reconcile_verification_head(&head);
16016            if waiting_on_gate {
16017                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16018                    head: head.clone(),
16019                    passed: true,
16020                    outcomes: Vec::new(),
16021                });
16022            }
16023            let cache = tmp.path().join("cache");
16024            runner.state.config.verify.e2e =
16025                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
16026            runner.state.config.verify.gate = vec!["true".into()];
16027            runner.state.config.graph.timeout_resource_wait = 1;
16028            let owner =
16029                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
16030            let guard =
16031                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
16032                    crate::cache::AcquireOutcome::Acquired(g) => g,
16033                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
16034                };
16035            runner.review_loop().await.unwrap();
16036            runner.gate().await.unwrap();
16037            assert!(!runner.state.gate_ran);
16038            assert_eq!(runner.state.gate_fixes.len(), 1);
16039            assert!(runner.state.resource_block.is_some());
16040            runner.merge().await.unwrap();
16041            assert!(runner.state.merge.is_none());
16042            drop(guard);
16043            runner.state = RunState::load(&runner.state.id).unwrap();
16044            // The driver clears the previous pass's resource stop on resume.
16045            runner.state.resource_block = None;
16046            runner.review_loop().await.unwrap();
16047            runner.gate().await.unwrap();
16048            assert!(runner.state.gate_allows_progress());
16049            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16050            assert_eq!(runner.state.gate_fixes.len(), 1);
16051        }
16052    }
16053
16054    #[cfg(unix)]
16055    #[tokio::test]
16056    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
16057        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16058        runner.sync_to_base().await.unwrap();
16059        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16060        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
16061        // Verification passes before pre_gate and fails on the mechanically changed head.
16062        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
16063        runner.review_loop().await.unwrap();
16064        runner.gate().await.unwrap();
16065        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16066        assert_ne!(before, after);
16067        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
16068        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
16069        assert!(!runner.state.gate_allows_progress());
16070        runner.merge().await.unwrap();
16071        assert!(runner.state.merge.is_none());
16072        assert!(
16073            runner
16074                .state
16075                .verification_history
16076                .iter()
16077                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
16078        );
16079
16080        // An incomplete review is still an independent hold on a changed head.
16081        runner.state.reviews[0].clean = false;
16082        runner.state.reviews[0].answered = 0;
16083        runner.state.config.verify.e2e = vec!["true".into()];
16084        runner.review_loop().await.unwrap();
16085        runner.gate().await.unwrap();
16086        runner.merge().await.unwrap();
16087        assert_eq!(runner.state.status, RunStatus::Blocked);
16088        assert!(runner.state.merge.is_none());
16089        assert!(
16090            runner
16091                .state
16092                .verification_blocking_cause()
16093                .unwrap()
16094                .contains("review blocked")
16095        );
16096    }
16097
16098    #[cfg(unix)]
16099    #[tokio::test]
16100    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
16101        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
16102        runner.state.gate_head = None;
16103        runner.state.config.verify.gate =
16104            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
16105        runner.review_loop().await.unwrap();
16106        runner.gate().await.unwrap();
16107        assert_ne!(
16108            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16109            old
16110        );
16111        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
16112        assert!(!runner.state.gate_allows_progress());
16113        assert_eq!(runner.state.status, RunStatus::Blocked);
16114        runner.merge().await.unwrap();
16115        assert!(runner.state.merge.is_none());
16116    }
16117
16118    #[cfg(unix)]
16119    #[tokio::test]
16120    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
16121        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16122        let mut old_review = review_round(true, 0, 1, 1, false, true);
16123        old_review.head = head.clone();
16124        old_review.verified_head = Some(head.clone());
16125        runner.state.reviews = vec![old_review];
16126        runner.state.gate = vec![CommandOutcome {
16127            command: "old gate".into(),
16128            code: None,
16129            output_tail: "timed out after 2400s".into(),
16130            duration_ms: 2_400_000,
16131            resource_blocked: false,
16132        }];
16133        runner.state.gate_ran = true;
16134        runner.state.gate_head = Some(head.clone());
16135        runner.state.land_approval = Some(crate::run::LandApproval {
16136            question: "old-head-approval".into(),
16137            head: head.clone(),
16138        });
16139        runner.state.authorize_base_sync(&task).unwrap();
16140        runner.sync_to_base().await.unwrap();
16141        assert!(!runner.state.base_sync_blocked());
16142        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16143        assert_ne!(head, fresh);
16144        assert!(runner.state.reviews.is_empty());
16145        assert!(!runner.state.gate_ran);
16146        let saved = &runner.state.base_sync_recovery.authorizations[0];
16147        assert_eq!(saved.reviews[0].head, head);
16148        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
16149        assert!(saved.gate_ran);
16150        assert!(!saved.gate[0].ok());
16151        runner.review_loop().await.unwrap();
16152        runner.gate().await.unwrap();
16153        assert_eq!(runner.state.reviews[0].head, fresh);
16154        assert_eq!(
16155            runner.state.reviews[0].verified_head.as_deref(),
16156            Some(fresh.as_str())
16157        );
16158        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
16159        assert!(runner.state.gate_ran);
16160        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16161        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
16162        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
16163        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16164        assert_eq!(
16165            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
16166            "retained change"
16167        );
16168    }
16169
16170    #[cfg(unix)]
16171    #[tokio::test]
16172    async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
16173        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16174        let queue = crate::queue::Queue::open();
16175        let mut owner = queue.get(&task).unwrap();
16176        owner.release_operator("test-resume", None);
16177        queue.put(&mut owner).unwrap();
16178        let generation = owner.resumption.as_ref().unwrap().generation;
16179        runner.state.task_instruction = Some(owner.instruction.clone());
16180        runner.state.driver_exited = true;
16181        runner.state.save().unwrap();
16182        let home = crate::run::home();
16183        let _claim = queue.claim(&task).unwrap();
16184        owner.resumption.as_mut().unwrap().at =
16185            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
16186        queue.put(&mut owner).unwrap();
16187        assert!(
16188            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16189                .await
16190                .is_err(),
16191            "an approval from before the stop grants nothing"
16192        );
16193        owner.resumption.as_mut().unwrap().at = Timestamp::now();
16194        queue.put(&mut owner).unwrap();
16195        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16196            .await
16197            .unwrap();
16198        assert!(
16199            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16200                .await
16201                .is_err()
16202        );
16203        runner.state = RunState::load(&runner.state.id).unwrap();
16204        assert_eq!(
16205            runner.state.base_sync_recovery.authorizations[0].generation,
16206            Some(generation)
16207        );
16208        runner.sync_to_base().await.unwrap();
16209        let repo = runner.state.repo.clone();
16210        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16211        git::git(&repo, &["checkout", "main"]).await.unwrap();
16212        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
16213        git::git(&repo, &["add", "."]).await.unwrap();
16214        git::git(&repo, &["commit", "-m", "move base again"])
16215            .await
16216            .unwrap();
16217        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16218        git::git(&repo, &["checkout", "fix/retained"])
16219            .await
16220            .unwrap();
16221        runner.finish_after_tally().await.unwrap();
16222        assert_eq!(runner.state.status, RunStatus::Blocked);
16223        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16224        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16225        assert!(!runner.state.base_recovery_pending());
16226        assert!(!crate::handover::can_resume(&runner.state));
16227        assert!(!runner.state.gate_ran);
16228        // Even replaying the generation with a later timestamp grants nothing.
16229        owner.resumption.as_mut().unwrap().at = Timestamp::now();
16230        queue.put(&mut owner).unwrap();
16231        assert!(
16232            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16233                .await
16234                .is_err()
16235        );
16236        runner.state = RunState::load(&runner.state.id).unwrap();
16237        runner.sync_to_base().await.unwrap();
16238        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16239        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
16240        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16241    }
16242
16243    #[cfg(unix)]
16244    #[tokio::test]
16245    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
16246        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
16247        runner.state.base_sync = None;
16248        runner.state.status = RunStatus::Blocked;
16249        let mut second = runner.state.candidates[0].clone();
16250        second.label = 'B';
16251        runner.state.candidates.push(second);
16252        assert!(runner.state.votes.is_empty());
16253        assert!(runner.state.deliberation.is_empty());
16254        runner.deliberate().await.unwrap();
16255        runner.vote().await.unwrap();
16256        assert_eq!(runner.state.status, RunStatus::Blocked);
16257        assert!(runner.state.votes.is_empty());
16258        assert!(runner.state.deliberation.is_empty());
16259    }
16260
16261    /// A minimal, git-free `Runner` at a given status — `settle_questions`
16262    /// reads nothing else off it.
16263    fn runner_at(status: RunStatus) -> Runner {
16264        let mut state = RunState::new(
16265            PathBuf::from("/nonexistent/repo"),
16266            "main".to_owned(),
16267            "deadbeef".to_owned(),
16268            "task".to_owned(),
16269            Config::default(),
16270        );
16271        // These fixtures share one home and save concurrently. Short clock
16272        // IDs can collide, making an ownership test read another test's run.
16273        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
16274        state.id = format!(
16275            "20000101-000000-{:04x}",
16276            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
16277        );
16278        state.status = status;
16279        Runner {
16280            state,
16281            roles: ResolvedRoles {
16282                implementers: Vec::new(),
16283                judges: Vec::new(),
16284                reviewers: Vec::new(),
16285                fixer: None,
16286                conductor: conductor(),
16287                implementer_roster: Vec::new(),
16288                judge_roster: Vec::new(),
16289                reviewer_roster: Vec::new(),
16290            },
16291            sem: Arc::new(Semaphore::new(1)),
16292            pause: Pause::new(),
16293            interrupt: Pause::new(),
16294        }
16295    }
16296
16297    #[tokio::test]
16298    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
16299        crate::run::pin_test_home();
16300        let mut runner = runner_at(RunStatus::Gating);
16301        runner.state.save().unwrap();
16302        let id = runner.state.id.clone();
16303        let passes = runner.state.passes.len();
16304        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16305        let mut claiming = Box::pin(runner.claim_driver());
16306        std::future::poll_fn(|cx| {
16307            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
16308            std::task::Poll::Ready(())
16309        })
16310        .await;
16311        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
16312        drop(owner);
16313        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
16314            .await
16315            .unwrap()
16316            .unwrap();
16317        assert_eq!(execution.runner.state.passes.len(), passes + 1);
16318        assert!(execution.runner.state.resource_block.is_none());
16319        assert_eq!(
16320            execution.runner.state.liveness(false),
16321            crate::run::Liveness::Live
16322        );
16323        drop(execution);
16324        assert!(RunState::load(&id).unwrap().driver_exited);
16325    }
16326
16327    #[tokio::test]
16328    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
16329        crate::run::pin_test_home();
16330        let mut runner = runner_at(RunStatus::Gating);
16331        runner.state.save().unwrap();
16332        // run.json can commit, but the subsequent lease metadata write fails.
16333        // Cleanup must run after releasing the short ownership lock.
16334        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
16335        assert!(runner.claim_driver().await.is_err());
16336        assert!(runner.state.driver_exited);
16337        assert_eq!(runner.state.driver_pid, None);
16338        let disk = RunState::load(&runner.state.id).unwrap();
16339        assert!(disk.driver_exited);
16340        assert_eq!(disk.driver_pid, None);
16341        assert_eq!(disk.status, RunStatus::Gating);
16342    }
16343
16344    #[tokio::test]
16345    async fn an_early_graph_error_releases_its_driver() {
16346        crate::run::pin_test_home();
16347        let mut runner = runner_at(RunStatus::Blocked);
16348        runner.state.config.graph.selected_implementer = Some("selected".into());
16349        runner.state.save().unwrap();
16350        assert!(runner.execute().await.is_err());
16351        let disk = RunState::load(&runner.state.id).unwrap();
16352        assert!(disk.driver_exited);
16353        assert_eq!(disk.driver_pid, None);
16354        assert_eq!(disk.status, RunStatus::Blocked);
16355    }
16356
16357    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
16358        crate::run::MergeRepair {
16359            question: "repair-question".into(),
16360            head: "old-head".into(),
16361            pr: "https://github.com/o/r/pull/42".into(),
16362            reviews: vec![review_round(false, 1, 1, 1, false, true)],
16363            gate: vec![outcome(Some(0), false)],
16364            gate_head: Some("old-head".into()),
16365            gate_fixes: Vec::new(),
16366            pre_gate: Vec::new(),
16367            phase,
16368            failure: None,
16369        }
16370    }
16371
16372    #[tokio::test]
16373    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
16374        crate::run::pin_test_home();
16375        let mut runner = runner_at(RunStatus::Landing);
16376        runner.state.merge_repairs.push(merge_repair_request(
16377            crate::run::MergeRepairPhase::Requested,
16378        ));
16379        runner.state.save().unwrap();
16380        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
16381        assert!(
16382            runner
16383                .execute()
16384                .await
16385                .unwrap_err()
16386                .to_string()
16387                .contains("already running")
16388        );
16389        assert_eq!(
16390            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16391            crate::run::MergeRepairPhase::Requested
16392        );
16393        drop(claim);
16394        let mut disk = runner.state.clone();
16395        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
16396        disk.save().unwrap();
16397        assert!(
16398            runner
16399                .execute()
16400                .await
16401                .unwrap_err()
16402                .to_string()
16403                .contains("decision changed")
16404        );
16405        assert_eq!(
16406            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16407            crate::run::MergeRepairPhase::Complete
16408        );
16409    }
16410
16411    #[test]
16412    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
16413        crate::run::pin_test_home();
16414        let mut runner = runner_at(RunStatus::Landing);
16415        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
16416        runner.state.reviews = request.reviews.clone();
16417        runner.state.gate = request.gate.clone();
16418        runner.state.gate_ran = true;
16419        runner.state.land_approval = Some(crate::run::LandApproval {
16420            question: "old".into(),
16421            head: "old-head".into(),
16422        });
16423        runner.state.merge_repairs.push(request);
16424        runner.begin_merge_repair_review(0).unwrap();
16425        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
16426        assert!(!runner.state.gate_status().ok());
16427        assert!(runner.state.land_approval.is_none());
16428        let disk = RunState::load(&runner.state.id).unwrap();
16429        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
16430        assert!(disk.merge_repairs[0].gate[0].ok());
16431        assert_eq!(
16432            disk.merge_repairs[0].phase,
16433            crate::run::MergeRepairPhase::Reviewing
16434        );
16435        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
16436        assert!(disk.reviews.is_empty());
16437    }
16438
16439    #[tokio::test]
16440    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
16441        crate::run::pin_test_home();
16442        for phase in [
16443            crate::run::MergeRepairPhase::Fixing,
16444            crate::run::MergeRepairPhase::Requested,
16445        ] {
16446            let mut runner = runner_at(RunStatus::Landing);
16447            runner.state.config.graph.land_rounds = 0;
16448            runner.state.merge_repairs.push(merge_repair_request(phase));
16449            runner.state.save().unwrap();
16450            assert!(!runner.repair_merge_review().await.unwrap());
16451            assert_eq!(runner.state.status, RunStatus::Blocked);
16452            assert_eq!(
16453                runner.state.merge_repairs[0].phase,
16454                crate::run::MergeRepairPhase::Failed
16455            );
16456            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
16457            assert!(
16458                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
16459                    "interrupted"
16460                } else {
16461                    "budget exhausted"
16462                })
16463            );
16464            assert!(runner.state.merge.is_none());
16465        }
16466    }
16467
16468    /// `park_here` folding in the reason `Pause::park_because` recorded -
16469    /// this is what lets an operator reading a run's events tell an
16470    /// interrupt-driven park from an ordinary shutdown park.
16471    #[test]
16472    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
16473        crate::run::pin_test_home();
16474        let mut runner = runner_at(RunStatus::Implementing);
16475        let interrupt = Pause::new();
16476        runner.watch_interrupt(interrupt.clone());
16477
16478        interrupt.park_because("task a1b2 asked to run first");
16479
16480        assert!(runner.park_here().expect("park_here"));
16481        assert!(runner.state.parked);
16482        let last = runner.state.events.last().expect("a park event");
16483        assert_eq!(last.node, "park");
16484        assert!(
16485            last.message.contains("task a1b2 asked to run first"),
16486            "expected the interrupt reason in {:?}",
16487            last.message
16488        );
16489    }
16490
16491    /// The cause is stored on the pass when it parks, and the first cause a
16492    /// shared pause was given is the one that stays.
16493    #[test]
16494    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
16495        crate::run::pin_test_home();
16496        let mut runner = runner_at(RunStatus::Implementing);
16497        runner.state.begin_pass();
16498        let shutdown = Pause::new();
16499        runner.on_pause(shutdown.clone());
16500        let upgrade = ParkCause::Upgrade {
16501            from: "0.121.0".to_owned(),
16502            to: Some("0.121.1".to_owned()),
16503        };
16504        shutdown.park_for(upgrade.clone());
16505        shutdown.park_for(ParkCause::Operator);
16506
16507        assert!(runner.park_here().expect("park_here"));
16508        assert_eq!(
16509            runner.state.passes[0].end,
16510            Some(crate::run::PassEnd::Parked { cause: upgrade })
16511        );
16512        let last = runner.state.events.last().expect("a park event");
16513        assert!(last.message.contains("0.121.0"), "{}", last.message);
16514    }
16515
16516    #[tokio::test]
16517    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
16518        crate::run::pin_test_home();
16519        let mut runner = runner_at(RunStatus::Reviewing);
16520        runner.state.save().unwrap();
16521        let mut execution = runner.claim_driver().await.expect("first execution");
16522        assert_eq!(execution.runner.state.passes.len(), 1);
16523        execution
16524            .driver
16525            .finish(&mut execution.runner.state)
16526            .await
16527            .unwrap();
16528    }
16529
16530    #[tokio::test]
16531    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
16532        crate::run::pin_test_home();
16533        let mut runner = runner_at(RunStatus::Reviewing);
16534        runner.state.config.agents = vec![conductor()];
16535        runner.state.config.graph.timeout_resource_wait = 0;
16536        runner.state.save().unwrap();
16537        let id = runner.state.id.clone();
16538        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16539        assert!(runner.claim_driver().await.is_err());
16540        drop(owner);
16541        drop(runner);
16542        let mut resumed = Runner::resume(&id).unwrap();
16543        assert_eq!(resumed.state.driver_claimed, Some(false));
16544        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
16545        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
16546        let mut execution = resumed
16547            .claim_driver()
16548            .await
16549            .expect("retry initial claim after reload");
16550        assert_eq!(execution.runner.state.driver_claimed, Some(true));
16551        assert_eq!(execution.runner.state.passes.len(), 1);
16552        execution
16553            .driver
16554            .finish(&mut execution.runner.state)
16555            .await
16556            .unwrap();
16557    }
16558
16559    #[tokio::test]
16560    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
16561        crate::run::pin_test_home();
16562        let mut runner = runner_at(RunStatus::Reviewing);
16563        runner.state.config.agents = vec![conductor()];
16564        runner.state.save().unwrap();
16565        let id = runner.state.id.clone();
16566        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
16567        let mut resumed = Runner::resume(&id).unwrap();
16568        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
16569        assert!(resumed.claim_driver().await.is_err());
16570        assert!(RunState::load(&id).unwrap().passes.is_empty());
16571        drop(lease);
16572        let mut execution = resumed.claim_driver().await.unwrap();
16573        execution
16574            .driver
16575            .finish(&mut execution.runner.state)
16576            .await
16577            .unwrap();
16578    }
16579
16580    #[tokio::test]
16581    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
16582        crate::run::pin_test_home();
16583        let mut runner = runner_at(RunStatus::Reviewing);
16584        let state = &mut runner.state;
16585        state.config.agents = vec![conductor()];
16586        state.driver_claimed = None;
16587        state.driver_pid = Some(std::process::id());
16588        state.driver_started_at = None;
16589        state.begin_pass();
16590        state.save().unwrap();
16591        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
16592        std::fs::write(
16593            state.dir().join("driver.json"),
16594            serde_json::to_vec(&serde_json::json!({
16595                "pid": state.driver_pid, "started_at": null,
16596                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
16597            }))
16598            .unwrap(),
16599        )
16600        .unwrap();
16601        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
16602        drop(lease);
16603        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
16604        let mut resumed = Runner::resume(&state.id).unwrap();
16605        assert!(!resumed.state.driver_exited);
16606        let mut execution = resumed
16607            .claim_driver()
16608            .await
16609            .expect("lease proves previous execution ended");
16610        assert_eq!(execution.runner.state.passes.len(), 2);
16611        execution
16612            .driver
16613            .finish(&mut execution.runner.state)
16614            .await
16615            .unwrap();
16616    }
16617
16618    #[tokio::test]
16619    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
16620        crate::run::pin_test_home();
16621        let mut runner = runner_at(RunStatus::Reviewing);
16622        let mut json = serde_json::to_value(&runner.state).unwrap();
16623        json.as_object_mut().unwrap().remove("driver_claimed");
16624        runner.state = serde_json::from_value(json).unwrap();
16625        assert_eq!(runner.state.driver_claimed, None);
16626        runner.state.save().unwrap();
16627        let mut execution = runner
16628            .claim_driver()
16629            .await
16630            .expect("the execution lease, not a missing marker, excludes a second driver");
16631        assert_eq!(execution.runner.state.passes.len(), 1);
16632        execution
16633            .driver
16634            .finish(&mut execution.runner.state)
16635            .await
16636            .unwrap();
16637    }
16638
16639    #[tokio::test]
16640    async fn driver_admission_refuses_a_proven_live_driver() {
16641        crate::run::pin_test_home();
16642        let mut runner = runner_at(RunStatus::Reviewing);
16643        let pid = std::process::id();
16644        runner.state.driver_pid = Some(pid);
16645        runner.state.driver_started_at = crate::proc::process_started_at(pid);
16646        assert!(runner.state.driver_started_at.is_some());
16647        runner.state.save().unwrap();
16648        let error = runner
16649            .claim_driver()
16650            .await
16651            .err()
16652            .expect("a live driver must refuse");
16653        assert!(error.to_string().contains("live driver"), "{error:#}");
16654        let saved = RunState::load(&runner.state.id).unwrap();
16655        assert_eq!(saved.driver_pid, Some(pid));
16656        assert_eq!(saved.status, RunStatus::Reviewing);
16657        assert!(saved.passes.is_empty());
16658    }
16659
16660    #[test]
16661    fn parking_resource_halt_persists_the_pass_once() {
16662        crate::run::pin_test_home();
16663        let mut runner = runner_at(RunStatus::Reviewing);
16664        runner.state.begin_pass();
16665        runner.state.resource_block = Some(crate::run::ResourceBlock {
16666            node: "fix".into(),
16667            kind: "shared build cache".into(),
16668            detail: "cancelled".into(),
16669            at: Timestamp::now(),
16670            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
16671        });
16672        let cause = ParkCause::Upgrade {
16673            from: "old".into(),
16674            to: Some("new".into()),
16675        };
16676        runner.pause.park_for(cause.clone());
16677        // Both direct fixer exits and graph resource boundaries must complete
16678        // parking, rather than returning a stale Reviewing record.
16679        runner.stop_for_resource().unwrap();
16680        assert!(runner.resource_halt().unwrap());
16681        assert!(runner.park_here().unwrap());
16682        let saved = RunState::load(&runner.state.id).unwrap();
16683        assert!(saved.parked);
16684        assert_eq!(saved.status, RunStatus::Reviewing);
16685        assert!(saved.passes[0].ended_at.is_some());
16686        assert_eq!(
16687            saved.passes[0].end,
16688            Some(crate::run::PassEnd::Parked {
16689                cause: cause.clone()
16690            })
16691        );
16692        assert_eq!(
16693            saved.resource_block.unwrap().cause,
16694            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
16695        );
16696        assert_eq!(
16697            saved
16698                .events
16699                .iter()
16700                .filter(|event| event.node == "park")
16701                .count(),
16702            1
16703        );
16704    }
16705
16706    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
16707    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
16708    /// cleared) must not make a *different* run - one only watching its own,
16709    /// unshared interrupt `Pause` - see itself as parked. If a future change
16710    /// ever collapsed these back into one handle, the interrupt scheduler
16711    /// would park every run for the rest of the daemon's life, not just the
16712    /// one it meant to interrupt.
16713    #[test]
16714    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
16715        crate::run::pin_test_home();
16716        let mut runner = runner_at(RunStatus::Implementing);
16717        let shutdown = Pause::new();
16718        runner.on_pause(shutdown.clone());
16719        let interrupt = Pause::new();
16720        runner.watch_interrupt(interrupt.clone());
16721
16722        // Nobody has asked for anything yet.
16723        assert!(!runner.park_here().expect("park_here"));
16724        assert!(!runner.state.parked);
16725
16726        // Only the interrupt handle fires; the shutdown handle stays clear.
16727        interrupt.park_because("test");
16728        assert!(!shutdown.parked());
16729        assert!(runner.park_here().expect("park_here"));
16730    }
16731
16732    /// The property every prior attempt at this feature failed to pin down:
16733    /// asking a run to park while one of its nodes has a real, in-flight
16734    /// async operation running (an agent call, in production) must not cut
16735    /// that operation short. `park_here` is only ever consulted *between*
16736    /// `execute`'s node calls - see its own doc - so nothing inside a node
16737    /// can observe a park request until the node itself returns. This proves
16738    /// that structurally, with real `tokio` concurrency and a channel
16739    /// handshake (never a sleep, which would only prove "usually", not
16740    /// "cannot"): the "node" below reports that it has genuinely started,
16741    /// and only then is the park requested; the node still has to be told to
16742    /// finish before `park_here` is ever called, exactly mirroring every
16743    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
16744    /// in `execute`.
16745    #[tokio::test]
16746    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
16747        crate::run::pin_test_home();
16748        let mut runner = runner_at(RunStatus::Implementing);
16749        let interrupt = Pause::new();
16750        runner.watch_interrupt(interrupt.clone());
16751
16752        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
16753        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
16754
16755        // Stands in for one node's in-flight agent call: it proves it has
16756        // genuinely started, then blocks - exactly as a spawned CLI process
16757        // does - until told to finish.
16758        let node = async move {
16759            started_tx.send(()).expect("send started");
16760            finish_rx.await.expect("recv finish");
16761            "node finished"
16762        };
16763
16764        let interrupter = async move {
16765            started_rx.await.expect("recv started");
16766            // The call is now genuinely in flight. Ask it to park.
16767            interrupt.park_because("higher-priority task waiting");
16768            // Nothing the node does can observe this yet - there is no
16769            // check inside it, by construction - so let the executor run
16770            // anything pending and then let the node finish on its own.
16771            tokio::task::yield_now().await;
16772            finish_tx.send(()).expect("send finish");
16773        };
16774
16775        let (node_result, ()) = tokio::join!(node, interrupter);
16776        assert_eq!(
16777            node_result, "node finished",
16778            "the in-flight call ran to completion"
16779        );
16780
16781        // Only now, at the boundary the real `execute` would check right
16782        // after this node, does the park take effect.
16783        assert!(runner.park_here().expect("park_here"));
16784        assert!(runner.state.parked);
16785    }
16786
16787    /// A run parked mid-competition carries every field it had accumulated
16788    /// through the exact same disk round-trip an ordinary resume uses -
16789    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
16790    /// Nothing about parking for an interrupt is a special case of that path;
16791    /// this is what proves it rather than assuming it.
16792    #[test]
16793    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
16794        crate::run::pin_test_home();
16795        let mut runner = runner_at(RunStatus::Judging);
16796        // `Runner::resume` re-resolves roles from the saved config, which
16797        // refuses an empty roster - give it the same minimal one `conductor`
16798        // itself uses.
16799        runner.state.config.agents = vec![conductor()];
16800        runner.state.candidates = vec![Candidate {
16801            index: 0,
16802            label: 'A',
16803            agent: "alpha".to_owned(),
16804            branch: "magi/x/A".to_owned(),
16805            worktree: PathBuf::from("/nonexistent/worktree"),
16806            summary: "did the thing".to_owned(),
16807            stat: "1 file changed".to_owned(),
16808            files: 1,
16809            commits: 1,
16810            empty: false,
16811            failed: None,
16812            verified_noop: None,
16813            duration_ms: 1234,
16814            folded: false,
16815        }];
16816        let run_id = runner.state.id.clone();
16817
16818        let interrupt = Pause::new();
16819        runner.watch_interrupt(interrupt.clone());
16820        interrupt.park_because("task c3d4 asked to run first");
16821        assert!(runner.park_here().expect("park_here"));
16822
16823        let resumed = Runner::resume(&run_id).expect("resume");
16824        assert_eq!(resumed.state.candidates.len(), 1);
16825        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
16826        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
16827        assert_eq!(resumed.state.status, runner.state.status);
16828        assert!(
16829            resumed.state.parked,
16830            "still parked until `execute` actually walks the graph again"
16831        );
16832        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
16833    }
16834
16835    /// A fresh open question on `run`, stored and handed back for assertions.
16836    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
16837        let mut q = ask::Question::new(
16838            run.to_owned(),
16839            "implement".to_owned(),
16840            "impl-A".to_owned(),
16841            "Which storage backend should the cache use?".to_owned(),
16842            String::new(),
16843            vec!["SQLite".to_owned(), "Redis".to_owned()],
16844        );
16845        store.put(&mut q).unwrap();
16846        q
16847    }
16848
16849    #[test]
16850    fn a_failed_runs_open_question_is_abandoned() {
16851        ask_test_home();
16852        let store = ask::Questions::open();
16853        let mut runner = runner_at(RunStatus::Failed);
16854        let run = runner.state.id.clone();
16855        let q = ask_open_question(&store, &run);
16856
16857        runner.settle_questions();
16858
16859        let back = store.get(&q.id).unwrap();
16860        assert!(
16861            !back.status.open(),
16862            "the seat that asked died with the run; nobody is left to read an answer"
16863        );
16864        assert!(
16865            back.detail.contains(&run) && back.detail.contains("failed"),
16866            "the reason names what the run became, not just that it is gone: {}",
16867            back.detail
16868        );
16869    }
16870
16871    #[test]
16872    fn a_merged_runs_open_question_is_abandoned_too() {
16873        ask_test_home();
16874        let store = ask::Questions::open();
16875        // A run that finishes cleanly still leaves nobody to read an answer -
16876        // this is not only a failure-path cleanup.
16877        for status in [RunStatus::Merged, RunStatus::Ready] {
16878            let mut runner = runner_at(status);
16879            let run = runner.state.id.clone();
16880            let q = ask_open_question(&store, &run);
16881
16882            runner.settle_questions();
16883
16884            let back = store.get(&q.id).unwrap();
16885            assert!(
16886                !back.status.open(),
16887                "{status:?} run's question must not outlive the run"
16888            );
16889        }
16890    }
16891
16892    #[test]
16893    fn a_still_resumable_runs_open_question_is_left_alone() {
16894        ask_test_home();
16895        let store = ask::Questions::open();
16896        // `Blocked` and `Stalled` can still be resumed — the candidates, the
16897        // review round and the seat sessions are all still on disk — so a
16898        // question asked mid-round may yet get a real answer from a real
16899        // resume. Sweeping it here would be exactly the failure mode this
16900        // whole feature exists to avoid on the other side.
16901        for status in [RunStatus::Blocked, RunStatus::Stalled] {
16902            let mut runner = runner_at(status);
16903            let run = runner.state.id.clone();
16904            let q = ask_open_question(&store, &run);
16905
16906            runner.settle_questions();
16907
16908            let back = store.get(&q.id).unwrap();
16909            assert!(
16910                back.status.open(),
16911                "{status:?} is still alive; the question must still be waiting"
16912            );
16913        }
16914    }
16915
16916    #[test]
16917    fn settle_questions_never_touches_an_already_answered_question() {
16918        ask_test_home();
16919        let store = ask::Questions::open();
16920        let mut runner = runner_at(RunStatus::Failed);
16921        let run = runner.state.id.clone();
16922        let mut q = ask_open_question(&store, &run);
16923        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
16924            .unwrap();
16925        store.put(&mut q).unwrap();
16926
16927        // Called twice, the way a crash-recovered daemon reclaim and the
16928        // graph's own cleanup both can for the same run — `abandon_for_run`
16929        // only ever touches what is still open, so this must be inert both
16930        // times, not merely the second.
16931        runner.settle_questions();
16932        runner.settle_questions();
16933
16934        let back = store.get(&q.id).unwrap();
16935        assert_eq!(
16936            back.status,
16937            ask::QuestionStatus::Answered,
16938            "a real answer is a decision on record, never overwritten by a sweep"
16939        );
16940    }
16941
16942    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
16943    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
16944    /// without merging, whose winner is still the operator's answer to read.
16945    /// Nothing previously called `fold_run` itself with a real `tally`, so
16946    /// this is the first test to pin down the one distinction the whole
16947    /// automatic-fold feature depends on: the winner's worktree and branch
16948    /// must survive, everything else sharing the run's worktree bay - a
16949    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
16950    /// second sweep treats every non-winner directory under the bay alike -
16951    /// must not.
16952    #[tokio::test]
16953    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
16954        crate::run::pin_test_home();
16955        let tmp = tempfile::tempdir().expect("tempdir");
16956        let repo = tmp.path().join("repo");
16957        std::fs::create_dir_all(&repo).unwrap();
16958        init_repo(&repo);
16959
16960        let mut config = Config::default();
16961        config.graph.worktree_root = Some(tmp.path().join("wt"));
16962
16963        let mut state = RunState::new(
16964            repo.clone(),
16965            "main".to_owned(),
16966            "deadbeef".to_owned(),
16967            "task".to_owned(),
16968            config,
16969        );
16970        let root = state.worktree_root();
16971        let wt_a = root.join("cand-A");
16972        let wt_b = root.join("cand-B");
16973        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
16974            .await
16975            .expect("worktree A");
16976        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
16977            .await
16978            .expect("worktree B");
16979
16980        state.candidates = vec![
16981            Candidate {
16982                index: 0,
16983                label: 'A',
16984                agent: "alpha".to_owned(),
16985                branch: "magi/x/A".to_owned(),
16986                worktree: wt_a.clone(),
16987                summary: String::new(),
16988                stat: String::new(),
16989                files: 0,
16990                commits: 0,
16991                empty: false,
16992                failed: None,
16993                verified_noop: None,
16994                duration_ms: 0,
16995                folded: false,
16996            },
16997            Candidate {
16998                index: 1,
16999                label: 'B',
17000                agent: "beta".to_owned(),
17001                branch: "magi/x/B".to_owned(),
17002                worktree: wt_b.clone(),
17003                summary: String::new(),
17004                stat: String::new(),
17005                files: 0,
17006                commits: 0,
17007                empty: false,
17008                failed: None,
17009                verified_noop: None,
17010                duration_ms: 0,
17011                folded: false,
17012            },
17013        ];
17014        state.tally = Some(Tally {
17015            first_choice: BTreeMap::from([('A', 1)]),
17016            borda: BTreeMap::new(),
17017            winner: 'A',
17018            rankings: 1,
17019            unanimous_initial: true,
17020            deliberated: false,
17021            changed_votes: 0,
17022            unanimous_final: true,
17023            tie_break: None,
17024            judges: 1,
17025            present: 1,
17026            quorum: 1,
17027            met_quorum: true,
17028            uncontested: None,
17029        });
17030        state.status = RunStatus::Ready;
17031
17032        fold_run(&mut state, false, &crate::run::home())
17033            .await
17034            .expect("fold_run");
17035
17036        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
17037        assert!(
17038            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17039            "the unmerged winner's branch survives"
17040        );
17041        assert!(
17042            !state.candidates[0].folded,
17043            "the winner is not marked folded"
17044        );
17045
17046        assert!(!wt_b.exists(), "the loser's worktree is removed");
17047        assert!(
17048            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
17049            "the loser's branch is removed"
17050        );
17051        assert!(state.candidates[1].folded, "the loser is marked folded");
17052    }
17053
17054    #[tokio::test]
17055    #[cfg(unix)]
17056    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
17057        use std::os::fd::AsRawFd as _;
17058        use std::time::Duration;
17059
17060        let tmp = tempfile::tempdir().unwrap();
17061        let repo = tmp.path().join("repo");
17062        std::fs::create_dir_all(&repo).unwrap();
17063        init_repo(&repo);
17064        let common = repo.join(".git");
17065        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
17066        let home = tmp.path().join("home");
17067        let mut config = Config::default();
17068        config.coordination = crate::config::test_coordination();
17069        config.graph.worktree_root = Some(tmp.path().join("wt"));
17070        config.graph.timeout_resource_wait = 1;
17071        let mut state = RunState::new(
17072            repo.clone(),
17073            "main".into(),
17074            "deadbeef".into(),
17075            "task".into(),
17076            config,
17077        );
17078        let root = state.worktree_root();
17079        let candidate = root.join("cand-A");
17080        let reviewer = root.join("review-1");
17081        let branch = state.branch_for('A');
17082        git::worktree_add_branch(&repo, &candidate, &branch, "main")
17083            .await
17084            .unwrap();
17085        git::worktree_add_detached(&repo, &reviewer, "main")
17086            .await
17087            .unwrap();
17088        git::acquire_worktree_config(&repo).await.unwrap();
17089        state.enabled_worktree_config = true;
17090        state.status = RunStatus::Ready;
17091        state.candidates.push(Candidate {
17092            index: 0,
17093            label: 'A',
17094            agent: "implement".into(),
17095            branch: branch.clone(),
17096            worktree: candidate.clone(),
17097            summary: String::new(),
17098            stat: String::new(),
17099            files: 0,
17100            commits: 0,
17101            empty: false,
17102            failed: None,
17103            verified_noop: None,
17104            duration_ms: 0,
17105            folded: false,
17106        });
17107        state.save_under(&home).unwrap();
17108        let config_before = std::fs::read(common.join("config")).unwrap();
17109        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
17110            .await
17111            .unwrap();
17112        // A separate OS handle bypasses the launcher's local gate, exactly as
17113        // an external updater does, and must still prevent all folding work.
17114        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
17115        assert_eq!(
17116            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
17117            0
17118        );
17119        let mut fold = Box::pin(fold_run(&mut state, true, &home));
17120        assert!(
17121            tokio::time::timeout(Duration::from_millis(200), &mut fold)
17122                .await
17123                .is_err()
17124        );
17125        assert!(candidate.exists());
17126        assert!(reviewer.exists());
17127        assert!(git::branch_exists(&repo, &branch).await.unwrap());
17128        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17129        assert_eq!(
17130            git::git(&repo, &["worktree", "list", "--porcelain"])
17131                .await
17132                .unwrap(),
17133            worktrees_before
17134        );
17135        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
17136            .await
17137            .unwrap()
17138            .unwrap_err();
17139        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
17140        drop(fold);
17141        assert!(candidate.exists() && reviewer.exists());
17142        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
17143        assert_eq!(
17144            state.resource_block.as_ref().unwrap().kind,
17145            "repository coordination"
17146        );
17147        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17148        drop(external);
17149        let mut fold = Box::pin(fold_run(&mut state, true, &home));
17150        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
17151            .await
17152            .unwrap()
17153            .unwrap();
17154        drop(fold);
17155        assert!(removed.contains(&branch));
17156        assert!(!root.exists());
17157        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
17158        assert!(state.candidates[0].folded);
17159        assert!(!state.enabled_worktree_config);
17160        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
17161        let enabled = git::git_raw(
17162            &repo,
17163            &["config", "--local", "--get", "extensions.worktreeConfig"],
17164        )
17165        .await
17166        .unwrap();
17167        assert!(!enabled.ok(), "the last config reference was released");
17168    }
17169
17170    /// A branch handed to a later run is that run's (and its pull request's):
17171    /// folding the run that released it must not delete it.
17172    #[tokio::test]
17173    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
17174        let tmp = tempfile::tempdir().expect("tempdir");
17175        let repo = tmp.path().join("repo");
17176        std::fs::create_dir_all(&repo).unwrap();
17177        init_repo(&repo);
17178        let home = tmp.path().join("home");
17179
17180        let mut config = Config::default();
17181        config.graph.worktree_root = Some(tmp.path().join("wt"));
17182        let mut state = RunState::new(
17183            repo.clone(),
17184            "main".to_owned(),
17185            "deadbeef".to_owned(),
17186            "task".to_owned(),
17187            config,
17188        );
17189        // The worktree is already gone (released); the branch survives.
17190        git::git(&repo, &["branch", "magi/x/A", "main"])
17191            .await
17192            .expect("branch");
17193        state.candidates = vec![Candidate {
17194            index: 0,
17195            label: 'A',
17196            agent: "alpha".to_owned(),
17197            branch: "magi/x/A".to_owned(),
17198            worktree: state.worktree_root().join("cand-A"),
17199            summary: String::new(),
17200            stat: String::new(),
17201            files: 0,
17202            commits: 0,
17203            empty: false,
17204            failed: None,
17205            verified_noop: None,
17206            duration_ms: 0,
17207            folded: true,
17208        }];
17209        state.released_to = Some("20260901-000000-new1".to_owned());
17210        state.released_branches = vec!["magi/x/A".to_owned()];
17211
17212        fold_run(&mut state, true, &home).await.expect("fold_run");
17213        state.status = RunStatus::Merged;
17214        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
17215        state.merge = Some(crate::run::MergeOutcome {
17216            mode: MergeMode::Pr,
17217            ok: true,
17218            detail: "confirmed merged".into(),
17219            empty: false,
17220        });
17221        fold_run(&mut state, true, &home)
17222            .await
17223            .expect("merged fold");
17224
17225        assert!(
17226            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17227            "the handed-over branch survives a fold"
17228        );
17229    }
17230
17231    /// A winner with nothing ahead of the base is caught before `gh` is ever
17232    /// asked for a pull request, and the message carries what the task's
17233    /// references resolved to.
17234    #[tokio::test]
17235    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
17236        let tmp = tempfile::tempdir().expect("tempdir");
17237        let repo = tmp.path().join("repo");
17238        std::fs::create_dir_all(&repo).unwrap();
17239        init_repo(&repo);
17240        let run = |args: &[&str]| {
17241            let out = std::process::Command::new("git")
17242                .quiet()
17243                .args(args)
17244                .current_dir(&repo)
17245                .output()
17246                .expect("spawn git");
17247            assert!(out.status.success(), "git {args:?}");
17248        };
17249        run(&["branch", "magi/x/A"]);
17250        run(&["checkout", "-q", "-b", "magi/x/B"]);
17251        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
17252        run(&["add", "-A"]);
17253        run(&["commit", "-q", "-m", "work"]);
17254        run(&["checkout", "-q", "main"]);
17255
17256        // A pull request is compared against the remote's base, so the
17257        // fixture needs one. Before it exists nothing can be read, and the
17258        // answer must be "not empty".
17259        let probe = RunState::new(
17260            repo.clone(),
17261            "main".to_owned(),
17262            "deadbeef".to_owned(),
17263            "task".to_owned(),
17264            Config::default(),
17265        );
17266        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
17267        let bare = tmp.path().join("origin.git");
17268        let out = std::process::Command::new("git")
17269            .quiet()
17270            .args(["init", "-q", "--bare"])
17271            .arg(&bare)
17272            .output()
17273            .expect("spawn git");
17274        assert!(out.status.success(), "git init --bare");
17275        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
17276        run(&["push", "-q", "origin", "main"]);
17277
17278        let mut state = RunState::new(
17279            repo.clone(),
17280            "main".to_owned(),
17281            "deadbeef".to_owned(),
17282            "task".to_owned(),
17283            Config::default(),
17284        );
17285        state.seeds = vec![refs::Seed {
17286            token: "magi/27b2/A".to_owned(),
17287            kind: refs::SeedKind::Unresolved,
17288            sha: String::new(),
17289            branch: true,
17290            detail: "no branch or commit named magi/27b2/A".to_owned(),
17291        }];
17292
17293        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
17294        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
17295        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
17296        let detail = empty_candidate_detail(&state, "main");
17297        assert!(detail.starts_with("empty candidate"), "{detail}");
17298        assert!(!detail.contains("magi/27b2/A"), "{detail}");
17299        assert!(detail.contains("--start-from"), "{detail}");
17300    }
17301
17302    /// `status == Ready` used to be read as "this is the harmless
17303    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
17304    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
17305    /// run's PR was closed without merging — and reentering `merge` with
17306    /// `mode` still `Pr` does not know the difference, so it pushed and
17307    /// opened a second pull request. `mode == Local` reproduces the same
17308    /// blind spot without a network call: reentry must not attempt another
17309    /// git merge once this node has already recorded an outcome.
17310    #[tokio::test]
17311    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
17312        let tmp = tempfile::tempdir().expect("tempdir");
17313        let repo = tmp.path().join("repo");
17314        std::fs::create_dir_all(&repo).unwrap();
17315        init_repo(&repo);
17316
17317        let mut config = Config::default();
17318        config.merge.mode = MergeMode::Local;
17319
17320        let mut state = RunState::new(
17321            repo.clone(),
17322            "main".to_owned(),
17323            "deadbeef".to_owned(),
17324            "task".to_owned(),
17325            config,
17326        );
17327        state.candidates = vec![Candidate {
17328            index: 0,
17329            label: 'A',
17330            agent: "alpha".to_owned(),
17331            branch: "does-not-exist".to_owned(),
17332            worktree: repo.clone(),
17333            summary: String::new(),
17334            stat: String::new(),
17335            files: 0,
17336            commits: 0,
17337            empty: false,
17338            failed: None,
17339            verified_noop: None,
17340            duration_ms: 0,
17341            folded: false,
17342        }];
17343        state.tally = Some(Tally {
17344            first_choice: BTreeMap::from([('A', 1)]),
17345            borda: BTreeMap::new(),
17346            winner: 'A',
17347            rankings: 1,
17348            unanimous_initial: true,
17349            deliberated: false,
17350            changed_votes: 0,
17351            unanimous_final: true,
17352            tie_break: None,
17353            judges: 0,
17354            present: 0,
17355            quorum: 0,
17356            met_quorum: true,
17357            uncontested: Some("only candidate A produced a change".to_owned()),
17358        });
17359        state.reviews = vec![ReviewRound {
17360            round: 1,
17361            head: "deadbeef".to_owned(),
17362            verified_head: None,
17363            verified_at: None,
17364            reviews: Vec::new(),
17365            e2e: Vec::new(),
17366            fix: None,
17367            blocking: 0,
17368            answered: 0,
17369            expected: 0,
17370            clean: true,
17371            verify_retried: false,
17372            e2e_deferred: false,
17373            e2e_defer_reason: None,
17374            progressed: false,
17375            vote_split: false,
17376            reconsideration: Vec::new(),
17377            verdict: None,
17378        }];
17379        state.gate = vec![CommandOutcome {
17380            command: "test".to_owned(),
17381            code: Some(0),
17382            output_tail: String::new(),
17383            duration_ms: 0,
17384            resource_blocked: false,
17385        }];
17386        state.gate_ran = true;
17387        // Reached its conclusion already — e.g. `land` closing the PR without
17388        // merging it, which (like the honest `MergeMode::None` path) leaves
17389        // `status` at `Ready`. The recorded outcome is what actually marks
17390        // this node done.
17391        state.status = RunStatus::Ready;
17392        state.merge = Some(MergeOutcome {
17393            mode: MergeMode::Local,
17394            ok: false,
17395            detail: "already concluded".to_owned(),
17396            empty: false,
17397        });
17398
17399        let mut runner = Runner {
17400            state,
17401            roles: ResolvedRoles {
17402                implementers: Vec::new(),
17403                judges: Vec::new(),
17404                reviewers: Vec::new(),
17405                fixer: None,
17406                conductor: conductor(),
17407                implementer_roster: Vec::new(),
17408                judge_roster: Vec::new(),
17409                reviewer_roster: Vec::new(),
17410            },
17411            sem: Arc::new(Semaphore::new(1)),
17412            pause: Pause::new(),
17413            interrupt: Pause::new(),
17414        };
17415
17416        runner.merge().await.expect("merge");
17417
17418        assert_eq!(
17419            runner.state.status,
17420            RunStatus::Ready,
17421            "a concluded run's status must not change on reentry"
17422        );
17423        assert_eq!(
17424            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17425            Some("already concluded"),
17426            "merge must not run again once the node already recorded an outcome"
17427        );
17428    }
17429
17430    /// `gate` leaves `state.gate_ran` false both before it has ever run and
17431    /// when its last attempt was resource-blocked (the shared build cache
17432    /// could not be acquired or confirmed fresh in time - see
17433    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
17434    /// `Vec` this also leaves behind used to read as "nothing failed" and let
17435    /// a run merge a tree the gate never actually checked - exactly the case
17436    /// a contended cache produces on every retry until it clears. `merge`
17437    /// must refuse until `gate` has actually recorded an attempt.
17438    #[tokio::test]
17439    async fn merge_refuses_a_gate_that_has_not_actually_run() {
17440        let tmp = tempfile::tempdir().expect("tempdir");
17441        let repo = tmp.path().join("repo");
17442        std::fs::create_dir_all(&repo).unwrap();
17443        init_repo(&repo);
17444
17445        let mut config = Config::default();
17446        config.merge.mode = MergeMode::Local;
17447
17448        let mut state = RunState::new(
17449            repo.clone(),
17450            "main".to_owned(),
17451            "deadbeef".to_owned(),
17452            "task".to_owned(),
17453            config,
17454        );
17455        state.candidates = vec![Candidate {
17456            index: 0,
17457            label: 'A',
17458            agent: "alpha".to_owned(),
17459            branch: "does-not-exist".to_owned(),
17460            worktree: repo.clone(),
17461            summary: String::new(),
17462            stat: String::new(),
17463            files: 0,
17464            commits: 0,
17465            empty: false,
17466            failed: None,
17467            verified_noop: None,
17468            duration_ms: 0,
17469            folded: false,
17470        }];
17471        state.tally = Some(Tally {
17472            first_choice: BTreeMap::from([('A', 1)]),
17473            borda: BTreeMap::new(),
17474            winner: 'A',
17475            rankings: 1,
17476            unanimous_initial: true,
17477            deliberated: false,
17478            changed_votes: 0,
17479            unanimous_final: true,
17480            tie_break: None,
17481            judges: 0,
17482            present: 0,
17483            quorum: 0,
17484            met_quorum: true,
17485            uncontested: Some("only candidate A produced a change".to_owned()),
17486        });
17487        state.reviews = vec![ReviewRound {
17488            round: 1,
17489            head: "deadbeef".to_owned(),
17490            verified_head: None,
17491            verified_at: None,
17492            reviews: Vec::new(),
17493            e2e: Vec::new(),
17494            fix: None,
17495            blocking: 0,
17496            answered: 0,
17497            expected: 0,
17498            clean: true,
17499            verify_retried: false,
17500            e2e_deferred: false,
17501            e2e_defer_reason: None,
17502            progressed: false,
17503            vote_split: false,
17504            reconsideration: Vec::new(),
17505            verdict: None,
17506        }];
17507        // The point: `gate` has not recorded anything yet.
17508        state.gate = Vec::new();
17509        state.gate_ran = false;
17510        state.status = RunStatus::Gating;
17511
17512        let mut runner = Runner {
17513            state,
17514            roles: ResolvedRoles {
17515                implementers: Vec::new(),
17516                judges: Vec::new(),
17517                reviewers: Vec::new(),
17518                fixer: None,
17519                conductor: conductor(),
17520                implementer_roster: Vec::new(),
17521                judge_roster: Vec::new(),
17522                reviewer_roster: Vec::new(),
17523            },
17524            sem: Arc::new(Semaphore::new(1)),
17525            pause: Pause::new(),
17526            interrupt: Pause::new(),
17527        };
17528
17529        runner.merge().await.expect("merge");
17530
17531        assert!(
17532            runner.state.merge.is_none(),
17533            "an empty gate must never be read as a passing one: {:?}",
17534            runner.state.merge
17535        );
17536    }
17537
17538    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
17539    /// commands configured and `merge.mode` is `none` (a review-only run).
17540    /// `gate` must still record a real attempt — zero commands, vacuously
17541    /// passed — rather than leaving `state.gate` empty in a way `merge`
17542    /// cannot tell apart from "never ran"; otherwise the run reaches
17543    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
17544    #[tokio::test]
17545    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
17546        let tmp = tempfile::tempdir().expect("tempdir");
17547        let repo = tmp.path().join("repo");
17548        std::fs::create_dir_all(&repo).unwrap();
17549        init_repo(&repo);
17550
17551        // Default config: `verify.gate` empty, `merge.mode` is `none`.
17552        let config = Config::default();
17553
17554        let mut state = RunState::new(
17555            repo.clone(),
17556            "main".to_owned(),
17557            "deadbeef".to_owned(),
17558            "task".to_owned(),
17559            config,
17560        );
17561        state.candidates = vec![Candidate {
17562            index: 0,
17563            label: 'A',
17564            agent: "alpha".to_owned(),
17565            branch: "does-not-exist".to_owned(),
17566            worktree: repo.clone(),
17567            summary: String::new(),
17568            stat: String::new(),
17569            files: 0,
17570            commits: 0,
17571            empty: false,
17572            failed: None,
17573            verified_noop: None,
17574            duration_ms: 0,
17575            folded: false,
17576        }];
17577        state.tally = Some(Tally {
17578            first_choice: BTreeMap::from([('A', 1)]),
17579            borda: BTreeMap::new(),
17580            winner: 'A',
17581            rankings: 1,
17582            unanimous_initial: true,
17583            deliberated: false,
17584            changed_votes: 0,
17585            unanimous_final: true,
17586            tie_break: None,
17587            judges: 0,
17588            present: 0,
17589            quorum: 0,
17590            met_quorum: true,
17591            uncontested: Some("only candidate A produced a change".to_owned()),
17592        });
17593        state.reviews = vec![ReviewRound {
17594            round: 1,
17595            head: "deadbeef".to_owned(),
17596            verified_head: None,
17597            verified_at: None,
17598            reviews: Vec::new(),
17599            e2e: Vec::new(),
17600            fix: None,
17601            blocking: 0,
17602            answered: 0,
17603            expected: 0,
17604            clean: true,
17605            verify_retried: false,
17606            e2e_deferred: false,
17607            e2e_defer_reason: None,
17608            progressed: false,
17609            vote_split: false,
17610            reconsideration: Vec::new(),
17611            verdict: None,
17612        }];
17613
17614        let mut runner = Runner {
17615            state,
17616            roles: ResolvedRoles {
17617                implementers: Vec::new(),
17618                judges: Vec::new(),
17619                reviewers: Vec::new(),
17620                fixer: None,
17621                conductor: conductor(),
17622                implementer_roster: Vec::new(),
17623                judge_roster: Vec::new(),
17624                reviewer_roster: Vec::new(),
17625            },
17626            sem: Arc::new(Semaphore::new(1)),
17627            pause: Pause::new(),
17628            interrupt: Pause::new(),
17629        };
17630
17631        runner.gate().await.expect("gate");
17632        assert!(
17633            runner.state.gate_ran,
17634            "zero configured commands is still a real attempt, not an unrun gate"
17635        );
17636        assert!(runner.state.gate.is_empty());
17637        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
17638        assert_ne!(
17639            runner.state.status,
17640            RunStatus::Blocked,
17641            "a gate with nothing to check must not read as failed"
17642        );
17643
17644        runner.merge().await.expect("merge");
17645        assert_eq!(
17646            runner.state.status,
17647            RunStatus::Ready,
17648            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
17649        );
17650    }
17651
17652    /// `Config::cache_dir` is derived from `verify.e2e` as well as
17653    /// `verify.gate` (so the e2e leg and the final gate never build against
17654    /// different directories). With zero `verify.gate` commands but a
17655    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
17656    /// that lease before discovering it had nothing to run - so a repo with
17657    /// no gate commands could come back `resource_blocked` (and therefore
17658    /// still `gate_ran == false`) on nothing but an unrelated run holding the
17659    /// cache, exactly the contention this run's own zero commands could
17660    /// never have touched. `gate` must recognise there is nothing to check
17661    /// before it ever asks for the lease.
17662    #[tokio::test]
17663    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
17664        crate::run::pin_test_home();
17665        let home = crate::run::home();
17666
17667        let tmp = tempfile::tempdir().expect("tempdir");
17668        let repo = tmp.path().join("repo");
17669        std::fs::create_dir_all(&repo).unwrap();
17670        init_repo(&repo);
17671        // Unique to this test, so holding its lease cannot collide with
17672        // another test sharing the same process-wide `home`.
17673        let cache_dir = tmp.path().join("target");
17674
17675        let mut config = Config::default();
17676        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
17677        // `verify.gate` stays empty (the default). Bounded so a regression
17678        // that does start waiting fails the test in seconds, not hangs it.
17679        config.graph.timeout_verify = Some(2);
17680        config.graph.timeout_resource_wait = 2;
17681
17682        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17683        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17684            .expect("no io error acquiring directly")
17685        {
17686            crate::cache::AcquireOutcome::Acquired(g) => g,
17687            crate::cache::AcquireOutcome::Busy(b) => {
17688                panic!("expected the direct acquire to win the lease first: {b:?}")
17689            }
17690        };
17691
17692        let mut state = RunState::new(
17693            repo.clone(),
17694            "main".to_owned(),
17695            "deadbeef".to_owned(),
17696            "task".to_owned(),
17697            config,
17698        );
17699        state.candidates = vec![Candidate {
17700            index: 0,
17701            label: 'A',
17702            agent: "alpha".to_owned(),
17703            branch: "does-not-exist".to_owned(),
17704            worktree: repo.clone(),
17705            summary: String::new(),
17706            stat: String::new(),
17707            files: 0,
17708            commits: 0,
17709            empty: false,
17710            failed: None,
17711            verified_noop: None,
17712            duration_ms: 0,
17713            folded: false,
17714        }];
17715        state.tally = Some(Tally {
17716            first_choice: BTreeMap::from([('A', 1)]),
17717            borda: BTreeMap::new(),
17718            winner: 'A',
17719            rankings: 1,
17720            unanimous_initial: true,
17721            deliberated: false,
17722            changed_votes: 0,
17723            unanimous_final: true,
17724            tie_break: None,
17725            judges: 0,
17726            present: 0,
17727            quorum: 0,
17728            met_quorum: true,
17729            uncontested: Some("only candidate A produced a change".to_owned()),
17730        });
17731        state.reviews = vec![ReviewRound {
17732            round: 1,
17733            head: "deadbeef".to_owned(),
17734            verified_head: None,
17735            verified_at: None,
17736            reviews: Vec::new(),
17737            e2e: Vec::new(),
17738            fix: None,
17739            blocking: 0,
17740            answered: 0,
17741            expected: 0,
17742            clean: true,
17743            verify_retried: false,
17744            e2e_deferred: false,
17745            e2e_defer_reason: None,
17746            progressed: false,
17747            vote_split: false,
17748            reconsideration: Vec::new(),
17749            verdict: None,
17750        }];
17751
17752        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
17753        state.reviews[0].e2e = vec![CommandOutcome {
17754            command: state.config.verify.e2e[0].clone(),
17755            code: Some(0),
17756            output_tail: String::new(),
17757            duration_ms: 0,
17758            resource_blocked: false,
17759        }];
17760        let mut runner = Runner {
17761            state,
17762            roles: ResolvedRoles {
17763                implementers: Vec::new(),
17764                judges: Vec::new(),
17765                reviewers: Vec::new(),
17766                fixer: None,
17767                conductor: conductor(),
17768                implementer_roster: Vec::new(),
17769                judge_roster: Vec::new(),
17770                reviewer_roster: Vec::new(),
17771            },
17772            sem: Arc::new(Semaphore::new(1)),
17773            pause: Pause::new(),
17774            interrupt: Pause::new(),
17775        };
17776
17777        let started = std::time::Instant::now();
17778        runner.gate().await.expect("gate");
17779        assert!(
17780            started.elapsed() < Duration::from_secs(1),
17781            "a gate with nothing to run must never wait on a lease it never needed"
17782        );
17783        assert!(
17784            runner.state.gate_ran,
17785            "zero commands is still a real, immediate attempt"
17786        );
17787        assert!(runner.state.gate.is_empty());
17788        assert_ne!(
17789            runner.state.status,
17790            RunStatus::Blocked,
17791            "must not read as resource-blocked on a lease it never asked for"
17792        );
17793    }
17794
17795    /// A separate process carries a live worker's environment throughout all
17796    /// verification entry points; the parallel harness never changes its env.
17797    #[cfg(unix)]
17798    #[tokio::test]
17799    async fn verification_is_standalone_inside_a_live_worker() {
17800        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
17801        if std::env::var_os(CHILD).is_none() {
17802            let home = tempfile::tempdir().unwrap();
17803            let generation = crate::worker::pin_current(home.path()).unwrap();
17804            let output = std::process::Command::new(std::env::current_exe().unwrap())
17805                .args([
17806                    "--exact",
17807                    "graph::tests::verification_is_standalone_inside_a_live_worker",
17808                    "--nocapture",
17809                ])
17810                .env(CHILD, "1")
17811                .env("MAGI_HOME", home.path())
17812                .env("MAGI_WORKER_HOME", home.path())
17813                .env("MAGI_WORKER_ID", "abcdef")
17814                .env("MAGI_WORKER_GENERATION", &generation.id)
17815                .env("MAGI_WORKER_EXE", &generation.executable)
17816                .env("MAGI_RUN", "verification-parent")
17817                .env("MAGI_NODE", "gate")
17818                .env("MAGI_SEAT", "worker-fixture")
17819                .env("MAGI_PROCESS_OWNER", "fixture-owner")
17820                .env("MAGI_NO_AUTOUPDATE", "0")
17821                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
17822                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
17823                .env(
17824                    "VERIFY_EXPECT_TOOLCHAIN",
17825                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
17826                )
17827                .env(
17828                    "VERIFY_EXPECT_CACHE",
17829                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
17830                )
17831                .output()
17832                .unwrap();
17833            assert!(
17834                output.status.success()
17835                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17836                "{}\n{}",
17837                String::from_utf8_lossy(&output.stdout),
17838                String::from_utf8_lossy(&output.stderr)
17839            );
17840            return;
17841        }
17842        let home = crate::run::home();
17843        let directory = home.join("workers/jobs/abcdef");
17844        std::fs::create_dir_all(&directory).unwrap();
17845        let job = crate::worker::Job {
17846            id: "abcdef".into(),
17847            sequence: 1,
17848            kind: "task".into(),
17849            key: "task:verification".into(),
17850            payload: serde_json::json!({}),
17851            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
17852            version: Some(env!("CARGO_PKG_VERSION").into()),
17853            status: crate::worker::Status::Running,
17854            result: None,
17855            pid: Some(std::process::id()),
17856            created_at: 0,
17857            finished_at: None,
17858        };
17859        std::fs::write(
17860            directory.join("job.json"),
17861            serde_json::to_vec(&job).unwrap(),
17862        )
17863        .unwrap();
17864        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
17865        let assert_worker = || {
17866            assert_eq!(
17867                crate::worker::current_job(&home).unwrap().unwrap().id,
17868                job.id
17869            );
17870            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
17871            assert_eq!(
17872                crate::worker::helper_executable(&home).unwrap(),
17873                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
17874            );
17875            assert!(
17876                crate::updater::Checker::new(&crate::config::Update {
17877                    mode: crate::config::UpdateMode::Notify,
17878                    interval: None,
17879                    auto_restart: false,
17880                })
17881                .is_none(),
17882                "the driver remains pinned, even while tests are standalone"
17883            );
17884        };
17885        assert_worker();
17886        let repo = home.join("repo");
17887        std::fs::create_dir_all(&repo).unwrap();
17888        init_repo(&repo);
17889        // Keep storage, seat/control context, containment and tool lookup.
17890        // No inherited values (or credentials) are printed on success.
17891        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
17892            test -z "${MAGI_WORKER_GENERATION+x}" &&
17893            test -z "${MAGI_WORKER_HOME+x}" &&
17894            test -z "${MAGI_WORKER_EXE+x}" &&
17895            test "$MAGI_HOME/repo" -ef . &&
17896            test "$MAGI_RUN" = verification-parent &&
17897            test "$MAGI_NODE" = gate &&
17898            test "$MAGI_SEAT" = worker-fixture &&
17899            test -n "$MAGI_PROCESS_OWNER" &&
17900            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
17901            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
17902            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
17903            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
17904        // Negative control: an ordinary child inherits the contamination.
17905        assert!(
17906            !std::process::Command::new("sh")
17907                .args(["-c", probe])
17908                .current_dir(&repo)
17909                .status()
17910                .unwrap()
17911                .success()
17912        );
17913        let unrelated_home = tempfile::tempdir().unwrap();
17914        assert!(
17915            crate::worker::current_job(unrelated_home.path()).is_err(),
17916            "an inherited worker id cannot be resolved in a unit fixture's home"
17917        );
17918        let mut runner = runner_at(RunStatus::Gating);
17919        runner.state.repo = repo.clone();
17920        runner.state.config.disk.min_free_bytes = 0;
17921        runner.state.config.coordination = crate::config::test_coordination();
17922        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
17923        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
17924        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
17925        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
17926        let winner = Candidate {
17927            index: 0,
17928            label: 'A',
17929            agent: "fixture".into(),
17930            branch: "main".into(),
17931            worktree: repo.clone(),
17932            summary: String::new(),
17933            stat: String::new(),
17934            files: 0,
17935            commits: 0,
17936            empty: false,
17937            failed: None,
17938            verified_noop: None,
17939            duration_ms: 0,
17940            folded: false,
17941        };
17942        let (e2e, retried, pids) = run_e2e_with_retry(
17943            &mut runner.state,
17944            &["sh".into(), "-c".into()],
17945            &[format!("{probe} && printf initial")],
17946            &repo,
17947            Duration::from_secs(10),
17948            "initial",
17949        )
17950        .await;
17951        assert!(!retried && pids.is_empty());
17952        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
17953        assert_eq!(e2e[0].output_tail, "initial");
17954        let gate = runner.run_gate(&winner).await.unwrap();
17955        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
17956        // An interrupted repair must recheck e2e from saved state. Landing
17957        // reuses that state and must run both pre-gate and gate again.
17958        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
17959            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
17960            passed: false,
17961            outcomes: Vec::new(),
17962        });
17963        runner.state.save().unwrap();
17964        let mut resumed = RunState::load(&runner.state.id).unwrap();
17965        let wait = crate::repo_lock::WaitPolicy::default();
17966        assert!(matches!(
17967            check_gate_repair_e2e(&mut resumed, &winner, &wait)
17968                .await
17969                .unwrap(),
17970            GateFix::Retry
17971        ));
17972        let saved = RunState::load(&resumed.id).unwrap();
17973        let repair = saved.gate_repair_e2e.unwrap();
17974        assert!(repair.passed);
17975        assert_eq!(repair.outcomes.len(), 1);
17976        assert!(repair.outcomes[0].ok());
17977        let landing = run_landing_gate(&mut resumed, &winner, &wait)
17978            .await
17979            .unwrap();
17980        assert_eq!(landing.len(), 1);
17981        assert!(landing[0].ok(), "{landing:?}");
17982        assert_eq!(
17983            std::fs::read_to_string(repo.join("phases")).unwrap(),
17984            "pre\ngate\ne2e\npre\ngate\n"
17985        );
17986        assert_worker();
17987    }
17988
17989    #[cfg(unix)]
17990    #[tokio::test]
17991    async fn verification_keeps_the_panic_and_the_full_output() {
17992        crate::run::pin_test_home();
17993        let tmp = tempfile::tempdir().unwrap();
17994        let mut config = Config::default();
17995        config.disk.min_free_bytes = 0;
17996        let mut state = RunState::new(
17997            tmp.path().to_owned(),
17998            "main".into(),
17999            "base".into(),
18000            "verification evidence".into(),
18001            config,
18002        );
18003        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
18004                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
18005                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
18006        let mut names = Vec::new();
18007        for _ in 0..2 {
18008            let (outcomes, _) = run_commands(
18009                &mut state,
18010                "gate",
18011                "gate",
18012                0,
18013                &["sh".into(), "-c".into()],
18014                &[script.into()],
18015                tmp.path(),
18016                Duration::from_secs(60),
18017            )
18018            .await;
18019            let tail = &outcomes[0].output_tail;
18020            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
18021            assert!(tail.contains("boom assertion"));
18022            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
18023            let line = tail.lines().next().unwrap();
18024            names.push(line.to_owned());
18025        }
18026        assert_ne!(names[0], names[1], "the second run must not overwrite");
18027        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
18028        assert!(full.len() > 500_000 && full.contains("boom assertion"));
18029        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
18030
18031        // Storage failures are evidence too; preserve the command's verdict.
18032        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
18033        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
18034        let (outcomes, _) = run_commands(
18035            &mut state,
18036            "gate",
18037            "gate",
18038            0,
18039            &["sh".into(), "-c".into()],
18040            &["echo failure-output; exit 7".into()],
18041            tmp.path(),
18042            Duration::from_secs(10),
18043        )
18044        .await;
18045        assert_eq!(outcomes[0].code, Some(7));
18046        assert!(state.events.iter().any(|e| {
18047            e.node == "gate" && e.message.contains("could not save the full gate output:")
18048        }));
18049    }
18050
18051    #[cfg(unix)]
18052    #[tokio::test]
18053    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
18054        crate::run::pin_test_home();
18055        let tmp = tempfile::tempdir().unwrap();
18056        let mut config = Config::default();
18057        config.disk.min_free_bytes = 0;
18058        let mut state = RunState::new(
18059            tmp.path().to_owned(),
18060            "main".into(),
18061            "base".into(),
18062            "verification evidence".into(),
18063            config,
18064        );
18065        let (outcomes, pids) = run_commands(
18066            &mut state,
18067            "gate",
18068            "gate",
18069            1,
18070            &["sh".into(), "-c".into()],
18071            &[
18072                "printf before-timeout; exec sleep 30".into(),
18073                "kill -TERM $$".into(),
18074                "exit 7".into(),
18075                "printf success".into(),
18076            ],
18077            tmp.path(),
18078            Duration::from_secs(1),
18079        )
18080        .await;
18081        wait_for_timed_out_children_to_die(&pids).await;
18082        assert_eq!(
18083            pids.len(),
18084            1,
18085            "unleased timeout retains its death-check PID"
18086        );
18087        assert_eq!(outcomes.len(), 4);
18088        assert_eq!(outcomes[0].code, None);
18089        assert!(outcomes[0].output_tail.contains("before-timeout"));
18090        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
18091        assert_eq!(outcomes[1].code, None);
18092        assert!(outcomes[1].output_tail.contains("command terminated:"));
18093        assert!(!outcomes[1].output_tail.contains("timed out"));
18094        assert_eq!(outcomes[2].code, Some(7));
18095        assert!(!outcomes[2].output_tail.contains("timed out"));
18096        assert_eq!(outcomes[3].code, Some(0));
18097        assert_eq!(outcomes[3].output_tail, "success");
18098        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
18099    }
18100
18101    /// The addendum's second gap: a `verify.gate` command running for real
18102    /// wall-clock time had nothing at all to show for it in `active` before
18103    /// `run_commands` learned to record it — a run could sit in `Gating` for
18104    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
18105    /// what was actually happening. Proven with a genuinely still-running
18106    /// command, not just a before/after check on the final state: a poller
18107    /// task reads the same `run.json` `gate()` is writing, the same way the
18108    /// phone or `magi show` would, while the shell command is still blocked
18109    /// on its own release marker.
18110    #[tokio::test]
18111    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
18112        crate::run::pin_test_home();
18113
18114        let tmp = tempfile::tempdir().expect("tempdir");
18115        let repo = tmp.path().join("repo");
18116        std::fs::create_dir_all(&repo).unwrap();
18117        init_repo(&repo);
18118
18119        let mut config = Config::default();
18120        config.verify.gate = vec![
18121            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
18122             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
18123                .to_owned(),
18124        ];
18125
18126        let mut state = RunState::new(
18127            repo.clone(),
18128            "main".to_owned(),
18129            "deadbeef".to_owned(),
18130            "task".to_owned(),
18131            config,
18132        );
18133        let run_id = state.id.clone();
18134        state.candidates = vec![Candidate {
18135            index: 0,
18136            label: 'A',
18137            agent: "alpha".to_owned(),
18138            branch: "does-not-exist".to_owned(),
18139            worktree: repo.clone(),
18140            summary: String::new(),
18141            stat: String::new(),
18142            files: 0,
18143            commits: 0,
18144            empty: false,
18145            failed: None,
18146            verified_noop: None,
18147            duration_ms: 0,
18148            folded: false,
18149        }];
18150        state.tally = Some(Tally {
18151            first_choice: BTreeMap::from([('A', 1)]),
18152            borda: BTreeMap::new(),
18153            winner: 'A',
18154            rankings: 1,
18155            unanimous_initial: true,
18156            deliberated: false,
18157            changed_votes: 0,
18158            unanimous_final: true,
18159            tie_break: None,
18160            judges: 0,
18161            present: 0,
18162            quorum: 0,
18163            met_quorum: true,
18164            uncontested: Some("only candidate A produced a change".to_owned()),
18165        });
18166        state.reviews = vec![ReviewRound {
18167            round: 1,
18168            head: "deadbeef".to_owned(),
18169            verified_head: None,
18170            verified_at: None,
18171            reviews: Vec::new(),
18172            e2e: Vec::new(),
18173            fix: None,
18174            blocking: 0,
18175            answered: 0,
18176            expected: 0,
18177            clean: true,
18178            verify_retried: false,
18179            e2e_deferred: false,
18180            e2e_defer_reason: None,
18181            progressed: false,
18182            vote_split: false,
18183            reconsideration: Vec::new(),
18184            verdict: None,
18185        }];
18186
18187        let mut runner = Runner {
18188            state,
18189            roles: ResolvedRoles {
18190                implementers: Vec::new(),
18191                judges: Vec::new(),
18192                reviewers: Vec::new(),
18193                fixer: None,
18194                conductor: conductor(),
18195                implementer_roster: Vec::new(),
18196                judge_roster: Vec::new(),
18197                reviewer_roster: Vec::new(),
18198            },
18199            sem: Arc::new(Semaphore::new(1)),
18200            pause: Pause::new(),
18201            interrupt: Pause::new(),
18202        };
18203
18204        let started_marker = repo.join("started.marker");
18205        let release_marker = repo.join("release.marker");
18206        let poller = tokio::spawn(async move {
18207            // Bounded so a regression that never records the task entry
18208            // fails this test in seconds instead of hanging the suite —
18209            // the same shape `a_park_requested_while_a_seat_is_mid_call_
18210            // does_not_cut_it_short` uses for the same reason.
18211            for _ in 0..100 {
18212                if started_marker.exists()
18213                    && let Ok(s) = crate::run::RunState::load(&run_id)
18214                    && let Some(a) = s.active.get("gate")
18215                {
18216                    std::fs::write(&release_marker, b"go").expect("release marker");
18217                    return Some(a.clone());
18218                }
18219                tokio::time::sleep(Duration::from_millis(50)).await;
18220            }
18221            None
18222        });
18223
18224        runner.gate().await.expect("gate");
18225        let captured = poller.await.expect("poller task");
18226        let captured = captured.expect(
18227            "the poller never saw a `gate` task entry in run.json while the command was \
18228             still blocked on its own release marker",
18229        );
18230
18231        assert_eq!(captured.task.as_deref(), Some("gate"));
18232        assert_eq!(captured.node, "gate");
18233        assert_eq!(captured.index, Some(1));
18234        assert_eq!(captured.total, Some(1));
18235        assert!(
18236            captured
18237                .command
18238                .as_deref()
18239                .is_some_and(|c| c.contains("started.marker")),
18240            "{captured:?}"
18241        );
18242
18243        assert!(
18244            runner.state.active.is_empty(),
18245            "the entry must be cleared once the command actually finished: {:?}",
18246            runner.state.active
18247        );
18248        assert!(runner.state.gate_ran);
18249        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18250    }
18251
18252    /// The hand-off over a blocking finding a reviewer rejected on leaves a
18253    /// record for `land`; one with only a Minor, or no reject, leaves none.
18254    #[tokio::test]
18255    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
18256        use crate::verdict::{Finding, ReviewVote, Severity};
18257        crate::run::pin_test_home();
18258        let tmp = tempfile::tempdir().expect("tempdir");
18259        let repo = tmp.path().join("repo");
18260        std::fs::create_dir_all(&repo).unwrap();
18261        init_repo(&repo);
18262
18263        for (severity, vote, expect) in [
18264            (Severity::Major, ReviewVote::Reject, true),
18265            (Severity::Minor, ReviewVote::Reject, false),
18266            (Severity::Major, ReviewVote::Approve, false),
18267        ] {
18268            let mut round = review_round(false, 1, 1, 1, false, true);
18269            round.reviews = vec![ReviewRecord {
18270                reviewer: 1,
18271                agent: "alpha".to_owned(),
18272                summary: String::new(),
18273                findings: vec![Finding {
18274                    id: "R1-1-1".to_owned(),
18275                    severity,
18276                    file: None,
18277                    line: None,
18278                    title: "t".to_owned(),
18279                    detail: String::new(),
18280                }],
18281                vote: Some(vote),
18282                failed: None,
18283                duration_ms: 0,
18284                attempts: 0,
18285            }];
18286            let mut state = RunState::new(
18287                repo.clone(),
18288                "main".to_owned(),
18289                "deadbeef".to_owned(),
18290                "task".to_owned(),
18291                Config::default(),
18292            );
18293            state.reviews = vec![round];
18294            let mut runner = Runner {
18295                state,
18296                roles: ResolvedRoles {
18297                    implementers: Vec::new(),
18298                    judges: Vec::new(),
18299                    reviewers: Vec::new(),
18300                    fixer: None,
18301                    conductor: conductor(),
18302                    implementer_roster: Vec::new(),
18303                    judge_roster: Vec::new(),
18304                    reviewer_roster: Vec::new(),
18305                },
18306                sem: Arc::new(Semaphore::new(1)),
18307                pause: Pause::new(),
18308                interrupt: Pause::new(),
18309            };
18310            let shell = runner.state.config.shell();
18311            runner
18312                .stop_reviewing("round budget spent", &shell, &repo)
18313                .await
18314                .expect("stop_reviewing");
18315            assert_eq!(runner.state.status, RunStatus::Gating);
18316            assert_eq!(
18317                runner.state.contested_handoff.is_some(),
18318                expect,
18319                "{severity:?} + {vote:?}"
18320            );
18321        }
18322    }
18323
18324    /// The shape the incident this whole fix responds to actually had: the
18325    /// round budget spent, the last round's own e2e blocked on the shared
18326    /// build cache (held here by a live pid — this test process — exactly
18327    /// `cache`'s own unit tests' pattern for "another owner, still alive"
18328    /// without forking a process). `stop_reviewing` must retry it — not
18329    /// silently leave the round looking untouched (the catch-up-only half of
18330    /// the bug), and not read the contention as a red `e2e` and block the
18331    /// run on it (the other half). Called directly, the same way
18332    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
18333    /// above exercises `gate`, so this never needs a real cargo build to
18334    /// reach: the lease is never released, so `with_cache_lease` never gets
18335    /// past acquiring it into anything that would need a real workspace.
18336    #[tokio::test]
18337    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
18338        crate::run::pin_test_home();
18339        let home = crate::run::home();
18340
18341        let tmp = tempfile::tempdir().expect("tempdir");
18342        let repo = tmp.path().join("repo");
18343        std::fs::create_dir_all(&repo).unwrap();
18344        init_repo(&repo);
18345        let head = crate::git::rev_parse(&repo, "HEAD")
18346            .await
18347            .expect("rev-parse");
18348        // Unique to this test, so holding its lease cannot collide with
18349        // another test sharing the same process-wide `home`.
18350        let cache_dir = tmp.path().join("target");
18351
18352        let mut config = Config::default();
18353        config.verify.e2e = vec![format!(
18354            "CARGO_TARGET_DIR='{}' test -f README.md",
18355            cache_dir.display()
18356        )];
18357        config.graph.review_rounds = 1;
18358        // Bounded so a regression that does start waiting fails the test in
18359        // seconds, not hangs it.
18360        config.graph.timeout_verify = Some(2);
18361        config.graph.timeout_resource_wait = 2;
18362
18363        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18364        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18365            .expect("no io error acquiring directly")
18366        {
18367            crate::cache::AcquireOutcome::Acquired(g) => g,
18368            crate::cache::AcquireOutcome::Busy(b) => {
18369                panic!("expected the direct acquire to win the lease first: {b:?}")
18370            }
18371        };
18372
18373        let mut state = RunState::new(
18374            repo.clone(),
18375            "main".to_owned(),
18376            head.clone(),
18377            "task".to_owned(),
18378            config,
18379        );
18380        state.candidates = vec![Candidate {
18381            index: 0,
18382            label: 'A',
18383            agent: "alpha".to_owned(),
18384            branch: "does-not-exist".to_owned(),
18385            worktree: repo.clone(),
18386            summary: String::new(),
18387            stat: String::new(),
18388            files: 0,
18389            commits: 0,
18390            empty: false,
18391            failed: None,
18392            verified_noop: None,
18393            duration_ms: 0,
18394            folded: false,
18395        }];
18396        state.tally = Some(Tally {
18397            first_choice: BTreeMap::from([('A', 1)]),
18398            borda: BTreeMap::new(),
18399            winner: 'A',
18400            rankings: 1,
18401            unanimous_initial: true,
18402            deliberated: false,
18403            changed_votes: 0,
18404            unanimous_final: true,
18405            tie_break: None,
18406            judges: 0,
18407            present: 0,
18408            quorum: 0,
18409            met_quorum: true,
18410            uncontested: Some("only candidate A produced a change".to_owned()),
18411        });
18412        // The round budget's last round, deferred: `needs_catchup_run`'s
18413        // other trigger. `stop_reviewing`'s retry machinery must treat this
18414        // exactly like a resource-blocked attempt once it actually runs.
18415        state.reviews = vec![ReviewRound {
18416            round: 1,
18417            head: head.clone(),
18418            verified_head: None,
18419            verified_at: None,
18420            reviews: Vec::new(),
18421            e2e: Vec::new(),
18422            fix: None,
18423            blocking: 1,
18424            answered: 1,
18425            expected: 1,
18426            clean: false,
18427            verify_retried: false,
18428            e2e_deferred: true,
18429            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
18430            progressed: false,
18431            vote_split: false,
18432            reconsideration: Vec::new(),
18433            verdict: None,
18434        }];
18435
18436        let mut runner = Runner {
18437            state,
18438            roles: ResolvedRoles {
18439                implementers: Vec::new(),
18440                judges: Vec::new(),
18441                reviewers: Vec::new(),
18442                fixer: None,
18443                conductor: conductor(),
18444                implementer_roster: Vec::new(),
18445                judge_roster: Vec::new(),
18446                reviewer_roster: Vec::new(),
18447            },
18448            sem: Arc::new(Semaphore::new(1)),
18449            pause: Pause::new(),
18450            interrupt: Pause::new(),
18451        };
18452
18453        let shell = runner.state.config.shell();
18454        runner
18455            .stop_reviewing("round budget spent", &shell, &repo)
18456            .await
18457            .expect("stop_reviewing");
18458
18459        let last = runner.state.reviews.last().expect("round record");
18460        assert_eq!(
18461            last.e2e_status(),
18462            E2eStatus::ResourceBlocked,
18463            "the shared cache is still held; the attempt must read as blocked, not deferred or \
18464             failed: {last:?}"
18465        );
18466        assert_eq!(
18467            last.verified_head.as_deref(),
18468            Some(head.as_str()),
18469            "which commit this attempt targeted is known even though nothing finished checking \
18470             it"
18471        );
18472        let first_attempt_at = last
18473            .verified_at
18474            .expect("when this attempt ran is known too");
18475        assert_ne!(
18476            runner.state.status,
18477            RunStatus::Blocked,
18478            "contention is evidence about the machine, not the patch — it must not settle the \
18479             run as blocked: {:?}",
18480            runner.state.status
18481        );
18482        assert!(
18483            !runner
18484                .state
18485                .events
18486                .iter()
18487                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
18488            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
18489            runner.state.events
18490        );
18491
18492        // The cache is still held: a later reentry must retry the same
18493        // round's verification again — not leave it looking exactly as
18494        // untouched as the first blocked attempt, which is indistinguishable
18495        // from never having tried again at all.
18496        runner
18497            .stop_reviewing("round budget spent", &shell, &repo)
18498            .await
18499            .expect("stop_reviewing retry");
18500        assert_eq!(
18501            runner.state.reviews.len(),
18502            1,
18503            "no new round was started: {:?}",
18504            runner.state.reviews
18505        );
18506        let last = runner.state.reviews.last().expect("round record");
18507        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
18508        assert!(
18509            last.verified_at.expect("still known") > first_attempt_at,
18510            "a second reentry must be a fresh attempt, not a stale copy of the first"
18511        );
18512        assert_ne!(runner.state.status, RunStatus::Blocked);
18513
18514        held.release();
18515    }
18516
18517    #[cfg(unix)]
18518    #[tokio::test]
18519    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
18520        ask_test_home();
18521        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
18522            let tmp = tempfile::tempdir().unwrap();
18523            let repo = tmp.path().join("repo");
18524            std::fs::create_dir(&repo).unwrap();
18525            init_repo(&repo);
18526            let run_git = |args: &[&str]| {
18527                let out = std::process::Command::new("git")
18528                    .args(args)
18529                    .current_dir(&repo)
18530                    .output()
18531                    .unwrap();
18532                assert!(
18533                    out.status.success(),
18534                    "{args:?}: {}",
18535                    String::from_utf8_lossy(&out.stderr)
18536                );
18537            };
18538            let origin = tmp.path().join("origin.git");
18539            run_git(&["init", "--bare", origin.to_str().unwrap()]);
18540            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
18541            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
18542            run_git(&["add", "spec.rs"]);
18543            run_git(&["commit", "-m", "initial spec"]);
18544            run_git(&["push", "origin", "main"]);
18545            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
18546            run_git(&["checkout", "-b", "fix/review"]);
18547            std::fs::write(
18548                repo.join("check.rs"),
18549                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
18550            )
18551            .unwrap();
18552            run_git(&["add", "check.rs"]);
18553            run_git(&["commit", "-m", "use spec"]);
18554            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
18555            run_git(&["checkout", "main"]);
18556            std::fs::write(
18557                repo.join("spec.rs"),
18558                "struct AgentSpec { max_turn_secs: u64 }\n",
18559            )
18560            .unwrap();
18561            run_git(&["add", "spec.rs"]);
18562            run_git(&["commit", "-m", "extend spec"]);
18563            run_git(&["push", "origin", "main"]);
18564            run_git(&["checkout", "fix/review"]);
18565
18566            let mut runner = runner_at(RunStatus::Reviewing);
18567            runner.state.repo = repo.clone();
18568            runner.state.base_commit = base;
18569            runner.state.pr_head = Some("fix/review".into());
18570            runner.state.config.graph.review_rounds = max_rounds;
18571            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
18572            runner.state.config.graph.sessions = false;
18573            runner.state.config.disk.min_free_bytes = 0;
18574            runner.state.config.verify.e2e =
18575                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
18576            runner.state.candidates = vec![Candidate {
18577                index: 0,
18578                label: 'A',
18579                agent: "fixture".into(),
18580                branch: "fix/review".into(),
18581                worktree: repo.clone(),
18582                summary: String::new(),
18583                stat: String::new(),
18584                files: 1,
18585                commits: 1,
18586                empty: false,
18587                failed: None,
18588                verified_noop: None,
18589                duration_ms: 0,
18590                folded: false,
18591            }];
18592            runner.state.tally = Some(Tally {
18593                first_choice: BTreeMap::new(),
18594                borda: BTreeMap::new(),
18595                winner: 'A',
18596                rankings: 0,
18597                unanimous_initial: true,
18598                deliberated: false,
18599                changed_votes: 0,
18600                unanimous_final: true,
18601                tie_break: None,
18602                judges: 0,
18603                present: 0,
18604                quorum: 0,
18605                met_quorum: true,
18606                uncontested: Some("review-only".into()),
18607            });
18608            let mut last = review_round(false, 0, 1, 1, false, false);
18609            last.head = reviewed.clone();
18610            last.e2e_deferred = true;
18611            last.e2e[0].resource_blocked = true;
18612            last.e2e[0].output_tail = "waiting for the shared build cache".into();
18613            if max_rounds == 1 {
18614                last.e2e.clear();
18615            }
18616            runner.state.reviews = vec![last];
18617            let calls = tmp.path().join("fix-calls");
18618            let mut fixer = spec("repair");
18619            fixer
18620                .env
18621                .insert("FIX_CALLS".into(), calls.display().to_string());
18622            fixer.command = vec!["sh".into(), "-c".into(), r#"
18623                echo fix >> "$FIX_CALLS"
18624                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
18625                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
18626                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
18627                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
18628            "#.into()];
18629            let mut reviewer = spec("reviewer");
18630            reviewer.command = vec![
18631                "sh".into(),
18632                "-c".into(),
18633                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
18634                    .into(),
18635            ];
18636            runner.roles.fixer = Some(vec![fixer]);
18637            runner.roles.reviewers = vec![reviewer];
18638
18639            runner.sync_to_base().await.unwrap();
18640            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
18641            assert_ne!(rebased, reviewed);
18642            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
18643            if reentry {
18644                // Simulate a crash after catch-up evidence was saved but before
18645                // dispatching the fixer. No dedicated pending-repair field exists.
18646                let shell = runner.state.config.shell();
18647                runner
18648                    .stop_unresolved_review("deferred verification", &shell, &repo)
18649                    .await
18650                    .unwrap();
18651                assert!(runner.catchup_repairable());
18652                assert!(!calls.exists());
18653                runner.state = RunState::load(&runner.state.id).unwrap();
18654                assert!(runner.catchup_repairable());
18655            }
18656            runner.review_loop().await.unwrap();
18657            let first = &runner.state.reviews[0];
18658            assert_eq!(first.head, reviewed);
18659            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
18660            assert!(
18661                first.verify_retried,
18662                "a reproduced build failure gets exactly one retry: {:?}",
18663                first.e2e
18664            );
18665            assert!(first.e2e[0].output_tail.contains("E0063"));
18666            assert!(!runner.catchup_repairable());
18667            if max_rounds == 2 {
18668                assert_eq!(
18669                    runner.state.status,
18670                    RunStatus::Gating,
18671                    "{}",
18672                    crate::report::run(&runner.state)
18673                );
18674                assert_eq!(runner.state.reviews.len(), 2);
18675                assert!(first.fix.as_ref().unwrap().committed);
18676                assert!(runner.state.reviews[1].clean);
18677                assert_eq!(
18678                    runner
18679                        .state
18680                        .reviews
18681                        .iter()
18682                        .filter(|r| r.fix.is_some())
18683                        .count(),
18684                    1
18685                );
18686                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
18687                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
18688            } else {
18689                assert_eq!(runner.state.status, RunStatus::Blocked);
18690                assert!(first.fix.is_none());
18691                assert!(!calls.exists());
18692                let recovery = crate::recovery::assess(
18693                    "prior",
18694                    Some(&runner.state),
18695                    &runner.landing_base(),
18696                    crate::recovery::Relation::Diverged,
18697                );
18698                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
18699                let reasons = recovery.reasons.join("\n");
18700                assert!(
18701                    reasons.contains(&reviewed) && reasons.contains(&rebased),
18702                    "{reasons}"
18703                );
18704                assert!(
18705                    reasons.contains("base sync to")
18706                        && reasons.contains("additional review is required"),
18707                    "{reasons}"
18708                );
18709            }
18710        }
18711    }
18712
18713    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
18714    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
18715    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
18716    /// own top-of-function fast path (`review_conclusion`) correctly reads
18717    /// this shape as `None` rather than guessing `Blocked`, and the loop's
18718    /// own `for` range is empty once the round budget is spent, so
18719    /// `review_loop` must retry the check itself rather than silently doing
18720    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
18721    /// above exercises directly, but through `review_loop`'s own entry point
18722    /// this time, proving the wiring between the two rather than just the
18723    /// retry logic in isolation.
18724    #[tokio::test]
18725    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
18726        crate::run::pin_test_home();
18727        let home = crate::run::home();
18728
18729        let tmp = tempfile::tempdir().expect("tempdir");
18730        let repo = tmp.path().join("repo");
18731        std::fs::create_dir_all(&repo).unwrap();
18732        init_repo(&repo);
18733        let head = crate::git::rev_parse(&repo, "HEAD")
18734            .await
18735            .expect("rev-parse");
18736        let cache_dir = tmp.path().join("target");
18737
18738        let mut config = Config::default();
18739        config.verify.e2e = vec![format!(
18740            "CARGO_TARGET_DIR='{}' test -f README.md",
18741            cache_dir.display()
18742        )];
18743        config.graph.review_rounds = 1;
18744        config.graph.timeout_verify = Some(2);
18745        config.graph.timeout_resource_wait = 2;
18746
18747        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18748        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18749            .expect("no io error acquiring directly")
18750        {
18751            crate::cache::AcquireOutcome::Acquired(g) => g,
18752            crate::cache::AcquireOutcome::Busy(b) => {
18753                panic!("expected the direct acquire to win the lease first: {b:?}")
18754            }
18755        };
18756
18757        let mut state = RunState::new(
18758            repo.clone(),
18759            "main".to_owned(),
18760            head.clone(),
18761            "task".to_owned(),
18762            config,
18763        );
18764        state.candidates = vec![Candidate {
18765            index: 0,
18766            label: 'A',
18767            agent: "alpha".to_owned(),
18768            branch: "does-not-exist".to_owned(),
18769            worktree: repo.clone(),
18770            summary: String::new(),
18771            stat: String::new(),
18772            files: 0,
18773            commits: 0,
18774            empty: false,
18775            failed: None,
18776            verified_noop: None,
18777            duration_ms: 0,
18778            folded: false,
18779        }];
18780        state.tally = Some(Tally {
18781            first_choice: BTreeMap::from([('A', 1)]),
18782            borda: BTreeMap::new(),
18783            winner: 'A',
18784            rankings: 1,
18785            unanimous_initial: true,
18786            deliberated: false,
18787            changed_votes: 0,
18788            unanimous_final: true,
18789            tie_break: None,
18790            judges: 0,
18791            present: 0,
18792            quorum: 0,
18793            met_quorum: true,
18794            uncontested: Some("only candidate A produced a change".to_owned()),
18795        });
18796        // The exact shape a prior process's `stop_reviewing` would have left
18797        // on disk: the round budget's last round, a real attempt already
18798        // made and already resource-blocked.
18799        state.reviews = vec![ReviewRound {
18800            round: 1,
18801            head: head.clone(),
18802            verified_head: Some(head.clone()),
18803            verified_at: Some(jiff::Timestamp::now()),
18804            reviews: Vec::new(),
18805            e2e: vec![CommandOutcome {
18806                command: format!(
18807                    "CARGO_TARGET_DIR='{}' test -f README.md",
18808                    cache_dir.display()
18809                ),
18810                code: None,
18811                output_tail: "waiting for the shared build cache".to_owned(),
18812                duration_ms: 0,
18813                resource_blocked: true,
18814            }],
18815            fix: None,
18816            blocking: 1,
18817            answered: 1,
18818            expected: 1,
18819            clean: false,
18820            verify_retried: false,
18821            e2e_deferred: false,
18822            e2e_defer_reason: None,
18823            progressed: false,
18824            vote_split: false,
18825            reconsideration: Vec::new(),
18826            verdict: None,
18827        }];
18828
18829        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
18830        let mut runner = Runner {
18831            state,
18832            roles: ResolvedRoles {
18833                implementers: Vec::new(),
18834                judges: Vec::new(),
18835                reviewers: Vec::new(),
18836                fixer: None,
18837                conductor: conductor(),
18838                implementer_roster: Vec::new(),
18839                judge_roster: Vec::new(),
18840                reviewer_roster: Vec::new(),
18841            },
18842            sem: Arc::new(Semaphore::new(1)),
18843            pause: Pause::new(),
18844            interrupt: Pause::new(),
18845        };
18846
18847        // The lease is still held throughout, so this reentry's own retry is
18848        // also contended — proving `review_loop` actually tried again (not
18849        // that it happened to succeed) is what the timestamp comparison
18850        // below is for.
18851        runner.review_loop().await.expect("review_loop");
18852
18853        assert_eq!(
18854            runner.state.reviews.len(),
18855            1,
18856            "no new round was started on top of the unresolved one: {:?}",
18857            runner.state.reviews
18858        );
18859        let last = &runner.state.reviews[0];
18860        assert_eq!(
18861            last.e2e_status(),
18862            E2eStatus::ResourceBlocked,
18863            "still contended: {last:?}"
18864        );
18865        assert!(
18866            last.verified_at.expect("still known") > first_attempt_at,
18867            "review_loop must have actually retried the check, not left it exactly as found"
18868        );
18869        assert_ne!(
18870            runner.state.status,
18871            RunStatus::Blocked,
18872            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
18873            runner.state.status
18874        );
18875
18876        held.release();
18877    }
18878
18879    #[tokio::test]
18880    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
18881        crate::run::pin_test_home();
18882        for status in [RunStatus::Blocked, RunStatus::Gating] {
18883            let mut runner = runner_at(status);
18884            runner.state.config.graph.land = true;
18885            runner.state.gate_ran = true;
18886            runner.state.merge = Some(MergeOutcome {
18887                mode: MergeMode::Pr,
18888                ok: false,
18889                detail: "the fixer produced no commit".into(),
18890                empty: false,
18891            });
18892            let url = "https://example.invalid/o/r/pull/1";
18893            runner.state.pr = Some(crate::run::PrRecord {
18894                url: url.into(),
18895                number: 1,
18896                state: "open".into(),
18897                checks: "red".into(),
18898                round: 1,
18899                rounds: 1,
18900                red_at_merge: Vec::new(),
18901            });
18902            runner.state.save().unwrap();
18903            // The nonexistent repository makes observation fail before a forge
18904            // command can run. We still must reach land, rather than silently
18905            // return from the graph with its old gate and merge records.
18906            runner.execute_graph().await.unwrap();
18907            assert_eq!(runner.state.status, RunStatus::Blocked);
18908            assert!(runner.state.land_sync_pending);
18909            assert!(runner.state.events.iter().any(|event|
18910                event.node == "land" && event.message == format!("watching {url}")));
18911            assert_eq!(
18912                runner.state.merge.as_ref().unwrap().detail,
18913                "the fixer produced no commit"
18914            );
18915        }
18916    }
18917
18918    #[tokio::test]
18919    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
18920        crate::run::pin_test_home();
18921        let tmp = tempfile::tempdir().expect("tempdir");
18922        let repo = tmp.path().join("repo");
18923        std::fs::create_dir_all(&repo).unwrap();
18924        init_repo(&repo);
18925
18926        let mut config = Config::default();
18927        config.merge.mode = MergeMode::Pr;
18928        config.graph.land = true;
18929        config.graph.land_approval = false;
18930
18931        let mut state = RunState::new(
18932            repo.clone(),
18933            "main".to_owned(),
18934            "deadbeef".to_owned(),
18935            "task".to_owned(),
18936            config,
18937        );
18938        state.candidates = vec![Candidate {
18939            index: 0,
18940            label: 'A',
18941            agent: "alpha".to_owned(),
18942            branch: "does-not-exist".to_owned(),
18943            worktree: repo.clone(),
18944            summary: String::new(),
18945            stat: String::new(),
18946            files: 0,
18947            commits: 0,
18948            empty: false,
18949            failed: None,
18950            verified_noop: None,
18951            duration_ms: 0,
18952            folded: false,
18953        }];
18954        state.tally = Some(Tally {
18955            first_choice: BTreeMap::from([('A', 1)]),
18956            borda: BTreeMap::new(),
18957            winner: 'A',
18958            rankings: 1,
18959            unanimous_initial: true,
18960            deliberated: false,
18961            changed_votes: 0,
18962            unanimous_final: true,
18963            tie_break: None,
18964            judges: 0,
18965            present: 0,
18966            quorum: 0,
18967            met_quorum: true,
18968            uncontested: Some("only candidate A produced a change".to_owned()),
18969        });
18970        state.reviews = vec![ReviewRound {
18971            round: 1,
18972            head: "deadbeef".to_owned(),
18973            verified_head: None,
18974            verified_at: None,
18975            reviews: Vec::new(),
18976            e2e: Vec::new(),
18977            fix: None,
18978            blocking: 0,
18979            answered: 0,
18980            expected: 0,
18981            clean: true,
18982            verify_retried: false,
18983            e2e_deferred: false,
18984            e2e_defer_reason: None,
18985            progressed: false,
18986            vote_split: false,
18987            reconsideration: Vec::new(),
18988            verdict: None,
18989        }];
18990        state.gate = vec![CommandOutcome {
18991            command: "test".to_owned(),
18992            code: Some(0),
18993            output_tail: String::new(),
18994            duration_ms: 0,
18995            resource_blocked: false,
18996        }];
18997        state.gate_ran = true;
18998        // A first pass through `merge` already pushed and opened this pull
18999        // request; `status` is `Landing` because a previous call into `land`
19000        // parked or was interrupted before it reached a terminal outcome.
19001        state.status = RunStatus::Landing;
19002        state.merge = Some(MergeOutcome {
19003            mode: MergeMode::Pr,
19004            ok: true,
19005            detail: "https://example.invalid/x/y/pull/1".to_owned(),
19006            empty: false,
19007        });
19008
19009        // The Landing-resume shortcut calls `run_land` directly rather than
19010        // through `merge`, which is exactly the call site that used to skip
19011        // `settle_questions` - see the fixture below.
19012        ask_test_home();
19013        // A real resume loads a persisted run; driver ownership is checked
19014        // against that record before the Landing shortcut is entered.
19015        state.save().expect("save landing run");
19016        let store = ask::Questions::open();
19017        let q = ask_open_question(&store, &state.id);
19018
19019        let mut runner = Runner {
19020            state,
19021            roles: ResolvedRoles {
19022                implementers: Vec::new(),
19023                judges: Vec::new(),
19024                reviewers: Vec::new(),
19025                fixer: None,
19026                conductor: conductor(),
19027                implementer_roster: Vec::new(),
19028                judge_roster: Vec::new(),
19029                reviewer_roster: Vec::new(),
19030            },
19031            sem: Arc::new(Semaphore::new(1)),
19032            pause: Pause::new(),
19033            interrupt: Pause::new(),
19034        };
19035
19036        // `execute`, not `merge` directly: the Landing-resume shortcut lives
19037        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
19038        // exactly because `review_loop` would otherwise clobber the marker
19039        // first.
19040        runner.execute().await.expect("execute");
19041
19042        assert_eq!(
19043            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19044            Some("https://example.invalid/x/y/pull/1"),
19045            "reentry must not push again or open a second pull request over the \
19046             one `land` is already watching"
19047        );
19048        assert_ne!(
19049            runner.state.status,
19050            RunStatus::Landing,
19051            "land could not actually reach the fake pull request, so it must \
19052             have given up rather than left the run silently parked forever"
19053        );
19054        // `land` could not reach the fake pull request, so it gave up into
19055        // `Blocked` - still resumable, so the question must not have been
19056        // swept just because this branch now also calls `settle_questions`.
19057        assert_eq!(runner.state.status, RunStatus::Blocked);
19058        assert!(
19059            store.get(&q.id).unwrap().status.open(),
19060            "Blocked is still alive; settle_questions must have been a no-op here"
19061        );
19062    }
19063
19064    fn state_with_round(round: ReviewRound) -> RunState {
19065        let mut s = RunState::new(
19066            PathBuf::from("/repo"),
19067            "main".to_owned(),
19068            "abc1234".to_owned(),
19069            "add retries".to_owned(),
19070            Config::default(),
19071        );
19072        s.reviews = vec![round];
19073        s
19074    }
19075
19076    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
19077        crate::verdict::Finding {
19078            id: id.to_owned(),
19079            severity,
19080            file: None,
19081            line: None,
19082            title: title.to_owned(),
19083            detail: String::new(),
19084        }
19085    }
19086
19087    #[test]
19088    fn pr_body_names_open_findings_and_declined_ones() {
19089        let round = ReviewRound {
19090            round: 2,
19091            head: "deadbee".to_owned(),
19092            verified_head: None,
19093            verified_at: None,
19094            reviews: vec![ReviewRecord {
19095                attempts: 0,
19096                reviewer: 1,
19097                agent: "alpha".to_owned(),
19098                summary: String::new(),
19099                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
19100                vote: None,
19101                failed: None,
19102                duration_ms: 0,
19103            }],
19104            e2e: vec![CommandOutcome {
19105                command: "cargo test".to_owned(),
19106                code: Some(0),
19107                output_tail: String::new(),
19108                duration_ms: 0,
19109                resource_blocked: false,
19110            }],
19111            verify_retried: false,
19112            e2e_deferred: false,
19113            e2e_defer_reason: None,
19114            fix: Some(FixRecord {
19115                agent: "alpha".to_owned(),
19116                addressed: Vec::new(),
19117                rejected: vec![crate::verdict::Rejection {
19118                    id: "R1-1-1".to_owned(),
19119                    why: "not reachable from any caller".to_owned(),
19120                }],
19121                notes: String::new(),
19122                committed: true,
19123                failed: None,
19124                duration_ms: 0,
19125                continuation: None,
19126            }),
19127            blocking: 0,
19128            answered: 1,
19129            expected: 1,
19130            clean: false,
19131            progressed: true,
19132            vote_split: false,
19133            reconsideration: Vec::new(),
19134            verdict: None,
19135        };
19136        let state = state_with_round(round);
19137        let body = pr_message(&state, 'A').body;
19138
19139        assert!(body.contains("add retries"), "the task must still be there");
19140        assert!(body.contains("R2-1-1"), "{body}");
19141        assert!(!body.contains("unused import"), "{body}");
19142        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
19143        assert!(
19144            !body.contains("not reachable from any caller"),
19145            "the reason it was declined: {body}"
19146        );
19147    }
19148
19149    #[test]
19150    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
19151        let round = ReviewRound {
19152            round: 1,
19153            head: "deadbee".to_owned(),
19154            verified_head: None,
19155            verified_at: None,
19156            reviews: vec![ReviewRecord {
19157                attempts: 0,
19158                reviewer: 1,
19159                agent: "alpha".to_owned(),
19160                summary: String::new(),
19161                findings: Vec::new(),
19162                vote: None,
19163                failed: None,
19164                duration_ms: 0,
19165            }],
19166            e2e: Vec::new(),
19167            verify_retried: false,
19168            e2e_deferred: false,
19169            e2e_defer_reason: None,
19170            fix: None,
19171            blocking: 0,
19172            answered: 1,
19173            expected: 1,
19174            clean: true,
19175            progressed: false,
19176            vote_split: false,
19177            reconsideration: Vec::new(),
19178            verdict: None,
19179        };
19180        let state = state_with_round(round);
19181        let body = pr_message(&state, 'A').body;
19182        assert!(!body.contains("Open review findings"), "{body}");
19183        assert!(!body.contains("Declined"), "{body}");
19184    }
19185
19186    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
19187        let mut state = RunState::new(
19188            PathBuf::from("/repo"),
19189            "main".to_owned(),
19190            "abc1234".to_owned(),
19191            instruction.to_owned(),
19192            Config::default(),
19193        );
19194        state.candidates.push(Candidate {
19195            index: 0,
19196            label: 'A',
19197            agent: "alpha".to_owned(),
19198            branch: "magi/x/A".to_owned(),
19199            worktree: PathBuf::from("/wt"),
19200            summary: summary.to_owned(),
19201            stat: String::new(),
19202            files: 1,
19203            commits: 1,
19204            empty: false,
19205            failed: None,
19206            verified_noop: None,
19207            folded: false,
19208            duration_ms: 0,
19209        });
19210        state
19211    }
19212
19213    fn review_state(subjects: &[&str]) -> RunState {
19214        let mut state = state_with_summary(
19215            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
19216            "",
19217        );
19218        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19219        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
19220        state
19221    }
19222
19223    /// A branch rebased onto a main that moved past the recorded
19224    /// `base_commit` is titled from its own first commit, never main's.
19225    #[tokio::test]
19226    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
19227        ask_test_home();
19228        let tmp = tempfile::tempdir().unwrap();
19229        let repo = tmp.path().join("repo");
19230        std::fs::create_dir_all(&repo).unwrap();
19231        init_repo(&repo);
19232        let origin = tmp.path().join("origin.git");
19233        let g = |dir: &Path, args: &[&str]| {
19234            let out = std::process::Command::new("git")
19235                .args(args)
19236                .current_dir(dir)
19237                .quiet()
19238                .output()
19239                .expect("spawn git");
19240            assert!(
19241                out.status.success(),
19242                "git {args:?}: {}",
19243                String::from_utf8_lossy(&out.stderr)
19244            );
19245        };
19246        g(
19247            tmp.path(),
19248            &[
19249                "clone",
19250                "--bare",
19251                "-q",
19252                repo.to_str().unwrap(),
19253                origin.to_str().unwrap(),
19254            ],
19255        );
19256        g(
19257            &repo,
19258            &["remote", "add", "origin", origin.to_str().unwrap()],
19259        );
19260        let c1 = git::rev_parse(&repo, "main").await.unwrap();
19261
19262        // Main moves on; the branch is built on top of the new main.
19263        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
19264        g(&repo, &["add", "-A"]);
19265        g(
19266            &repo,
19267            &["commit", "-q", "-m", "chore(deps): update a crate"],
19268        );
19269        g(&repo, &["push", "-q", "origin", "main"]);
19270        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
19271        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19272        g(&repo, &["add", "-A"]);
19273        g(
19274            &repo,
19275            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
19276        );
19277        g(&repo, &["checkout", "-q", "main"]);
19278
19279        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
19280        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
19281        // Without a readable tracking ref the recorded base's merge base is used.
19282        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
19283        assert_eq!(fallback, c1);
19284
19285        let mut state = review_state(&[
19286            "chore(deps): update a crate",
19287            "fix(daemon): apply a chosen action",
19288        ]);
19289        state.repo = repo.clone();
19290        state.base_branch = "main".to_owned();
19291        state.base_commit = c1;
19292        refresh_reviewed_commits(&mut state, "feat/own").await;
19293        assert_eq!(
19294            state.reviewed_commits,
19295            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
19296        );
19297        assert_eq!(
19298            review_title(&state).as_deref(),
19299            Some("fix(daemon): apply a chosen action")
19300        );
19301        assert_eq!(
19302            leaked_subjects(&state, "feat/own").await,
19303            Some(vec!["chore(deps): update a crate".to_owned()])
19304        );
19305        // A stale tracking ref is still used when the fetch fails, but the
19306        // leak list is withheld.
19307        state.config.merge.remote = "nowhere".to_owned();
19308        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
19309    }
19310
19311    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
19312    /// with `origin/main` one commit behind it, and a summarizing agent that
19313    /// prints whatever `reply` holds.
19314    struct SummaryFixture {
19315        _tmp: tempfile::TempDir,
19316        repo: PathBuf,
19317        reply: PathBuf,
19318        runner: Runner,
19319    }
19320
19321    fn summary_fixture() -> SummaryFixture {
19322        ask_test_home();
19323        let tmp = tempfile::tempdir().unwrap();
19324        let repo = tmp.path().join("repo");
19325        std::fs::create_dir_all(&repo).unwrap();
19326        init_repo(&repo);
19327        let origin = tmp.path().join("origin.git");
19328        let g = |args: &[&str]| {
19329            let out = std::process::Command::new("git")
19330                .args(args)
19331                .current_dir(&repo)
19332                .quiet()
19333                .output()
19334                .expect("spawn git");
19335            assert!(
19336                out.status.success(),
19337                "git {args:?}: {}",
19338                String::from_utf8_lossy(&out.stderr)
19339            );
19340        };
19341        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
19342        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
19343        g(&["push", "-q", "origin", "main"]);
19344        g(&["checkout", "-q", "-b", "feat/own"]);
19345        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19346        g(&["add", "-A"]);
19347        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
19348        let reply = tmp.path().join("reply.txt");
19349        std::fs::write(
19350            &reply,
19351            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
19352        )
19353        .unwrap();
19354        let mut runner = runner_at(RunStatus::Gating);
19355        let mut state = review_state(&["feat(own): add the own file"]);
19356        state.repo = repo.clone();
19357        state.base_branch = "main".to_owned();
19358        state.candidates[0].branch = "feat/own".to_owned();
19359        state.candidates[0].worktree = repo.clone();
19360        let mut author = spec("alpha");
19361        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
19362        author
19363            .env
19364            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
19365        state.config.agents = vec![author];
19366        runner.state = state;
19367        SummaryFixture {
19368            _tmp: tmp,
19369            repo,
19370            reply,
19371            runner,
19372        }
19373    }
19374
19375    async fn posted(f: &mut SummaryFixture) -> PrMessage {
19376        let winner = f.runner.state.candidates[0].clone();
19377        f.runner
19378            .guarded_pr_message(&winner, None, true)
19379            .await
19380            .unwrap()
19381            .expect("an English description never parks")
19382    }
19383
19384    fn clean_round_with_minor_findings() -> ReviewRound {
19385        let mut round = passing_round(2, None);
19386        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
19387        round.reviews = vec![ReviewRecord {
19388            attempts: 0,
19389            reviewer: 1,
19390            agent: "alpha".into(),
19391            summary: "内部レビュー".into(),
19392            findings: vec![
19393                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
19394                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
19395            ],
19396            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
19397            failed: None,
19398            duration_ms: 0,
19399        }];
19400        round
19401    }
19402
19403    #[tokio::test]
19404    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
19405        for review_only in [false, true] {
19406            let mut f = summary_fixture();
19407            if !review_only {
19408                f.runner.state.candidates[0].agent = "alpha".into();
19409            }
19410            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
19411            assert!(f.runner.state.reviews[0].clean);
19412            assert!(f.runner.state.reviews[0].fix.is_none());
19413            assert!(f.runner.state.open_findings().is_empty());
19414            assert_eq!(
19415                crate::pr_outcome::required_finding_ids(&f.runner.state),
19416                ["R2-1-1", "R2-1-2"]
19417            );
19418            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19419            let evidence = crate::pr_outcome::summary_evidence(
19420                &f.runner.state,
19421                crate::pr_outcome::FinalHead::At(&head),
19422            );
19423            assert!(
19424                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
19425            );
19426            // The ordinary reply omits both IDs: no cached description is adopted.
19427            let omitted = posted(&mut f).await;
19428            assert!(omitted.body.contains("No description"));
19429            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
19430            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
19431            assert!(f.runner.state.change_summary.is_none());
19432            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
19433            let described = posted(&mut f).await;
19434            assert!(described.body.contains("improve documentation for callers"));
19435            assert!(described.body.contains("remaining edge case"));
19436            assert!(described.body.contains("## Open review findings"));
19437            assert!(!described.body.contains("未修正"));
19438        }
19439    }
19440
19441    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
19442        let mut f = summary_fixture();
19443        if !review_only {
19444            f.runner.state.candidates[0].agent = "alpha".into();
19445        }
19446        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19447        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
19448        fixed
19449            .fix
19450            .as_mut()
19451            .unwrap()
19452            .rejected
19453            .push(crate::verdict::Rejection {
19454                id: "R1-1-1".into(),
19455                why: "呼び出されないコードのため辞退".into(),
19456            });
19457        let mut clean = clean_round_with_minor_findings();
19458        clean.verified_head = Some(head.clone());
19459        f.runner.state.reviews = vec![fixed, clean];
19460        ok_gate(&mut f.runner.state, Some(&head));
19461        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
19462        f.runner.state.config.agents.push(scripted("beta", reply));
19463        f.runner.state.config.roles.language_judge =
19464            Some(crate::config::AgentChoice::One("beta".into()));
19465        f
19466    }
19467
19468    #[tokio::test]
19469    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
19470        for review_only in [false, true] {
19471            let mut f = rewrite_fixture_with_findings(review_only,
19472                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
19473            let m = posted(&mut f).await;
19474            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19475            let expected =
19476                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
19477            assert!(m.body.contains(&expected));
19478            assert!(m.body.contains("## Review rounds"));
19479            assert!(
19480                m.body
19481                    .contains("Gate (`verify.gate`): passed on the final commit")
19482            );
19483            assert!(
19484                m.body
19485                    .contains("Tests (`verify.e2e`): passed on the final commit")
19486            );
19487            let c = f.runner.state.change_summary.as_ref().unwrap();
19488            assert!(
19489                m.body
19490                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
19491            );
19492            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19493            assert!(crate::pr_outcome::covers_required_findings(
19494                &f.runner.state,
19495                &m.body
19496            ));
19497            assert!(m.body.contains("documentation remains incomplete"));
19498            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
19499            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
19500            assert!(f.runner.state.github_text.is_none());
19501        }
19502    }
19503
19504    #[tokio::test]
19505    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
19506        for review_only in [false, true] {
19507            for reply in [
19508                "TITLE: fix: public description\nA short English description.",
19509                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
19510            ] {
19511                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
19512                let winner = f.runner.state.candidates[0].clone();
19513                let posted = f
19514                    .runner
19515                    .guarded_pr_message(&winner, None, true)
19516                    .await
19517                    .unwrap();
19518                assert!(posted.is_none());
19519                assert!(f.runner.state.parked);
19520                let withheld = f.runner.state.github_text.as_ref().unwrap();
19521                assert!(
19522                    withheld.rewrite.is_none(),
19523                    "an incomplete rewrite must not be selectable"
19524                );
19525                let q = ask::Questions::open()
19526                    .get(withheld.question.as_ref().unwrap())
19527                    .unwrap();
19528                assert!(
19529                    !q.choices
19530                        .contains(&crate::github_text::USE_REWRITE.to_owned())
19531                );
19532                assert!(
19533                    q.detail.contains("R1-1-1")
19534                        && q.detail.contains("R2-1-1")
19535                        && q.detail.contains("R2-1-2")
19536                );
19537            }
19538        }
19539    }
19540
19541    #[tokio::test]
19542    async fn japanese_reports_stay_local_in_both_public_run_modes() {
19543        for review_only in [false, true] {
19544            let mut f = summary_fixture();
19545            if !review_only {
19546                f.runner.state.candidates[0].agent = "alpha".into();
19547            }
19548            f.runner.state.instruction = "内部タスクの説明".into();
19549            f.runner.state.candidates[0].summary = "内部の候補報告".into();
19550            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19551            f.runner.state.reviews = vec![
19552                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
19553                round_with_notes(2, Some("二回目の内部修正報告です。")),
19554                passing_round(3, Some(&head)),
19555            ];
19556            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
19557            ok_gate(&mut f.runner.state, Some(&head));
19558            let captured = f.repo.join("public-prompt.txt");
19559            let author = &mut f.runner.state.config.agents[0];
19560            author.command = vec![
19561                "sh".into(),
19562                "-c".into(),
19563                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19564            ];
19565            author.env.insert(
19566                "PROMPT_FILE".into(),
19567                captured.to_string_lossy().into_owned(),
19568            );
19569            let m = posted(&mut f).await;
19570            assert!(m.body.contains("Adds `own.txt`"));
19571            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
19572            assert!(m.body.contains("R1-1-1"));
19573            assert!(
19574                m.body
19575                    .contains("Gate (`verify.gate`): passed on the final commit")
19576            );
19577            assert!(
19578                m.body
19579                    .contains("Tests (`verify.e2e`): passed on the final commit")
19580            );
19581            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19582            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
19583            let prompt = std::fs::read_to_string(captured).unwrap();
19584            assert!(prompt.contains(&format!(
19585                "git diff {} {head}",
19586                f.runner.state.change_summary.as_ref().unwrap().base
19587            )));
19588            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
19589            assert_eq!(
19590                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
19591                "二回目の内部修正報告です。"
19592            );
19593        }
19594    }
19595
19596    #[tokio::test]
19597    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
19598        for review_only in [false, true] {
19599            let mut f = summary_fixture();
19600            if !review_only {
19601                f.runner.state.candidates[0].agent = "alpha".into();
19602            }
19603            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
19604            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
19605            std::fs::write(&f.reply, "no usable description").unwrap();
19606            let failed = posted(&mut f).await;
19607            assert!(
19608                failed
19609                    .body
19610                    .contains("No description of what this change does could be generated")
19611            );
19612            assert!(failed.body.is_ascii());
19613            assert!(f.runner.state.github_text.is_none());
19614            assert!(f.runner.state.change_summary.is_none());
19615            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19616            let evidence = crate::pr_outcome::summary_evidence(
19617                &f.runner.state,
19618                crate::pr_outcome::FinalHead::At(&head),
19619            );
19620            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
19621            assert!(
19622                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
19623                    .exists()
19624            );
19625            f.runner.state =
19626                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
19627            std::fs::write(
19628                &f.reply,
19629                "TITLE: feat(own): add the own file\nAdds the committed own file.",
19630            )
19631            .unwrap();
19632            let retried = posted(&mut f).await;
19633            assert!(retried.body.contains("Adds the committed own file."));
19634            assert_eq!(
19635                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
19636                "日本語の修正報告"
19637            );
19638        }
19639    }
19640
19641    #[tokio::test]
19642    async fn same_head_verification_and_findings_invalidate_public_prose() {
19643        let mut f = summary_fixture();
19644        posted(&mut f).await;
19645        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19646        let original = f
19647            .runner
19648            .state
19649            .change_summary
19650            .as_ref()
19651            .unwrap()
19652            .evidence
19653            .clone();
19654        ok_gate(&mut f.runner.state, Some(&head));
19655        std::fs::write(
19656            &f.reply,
19657            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
19658        )
19659        .unwrap();
19660        let verified = posted(&mut f).await;
19661        assert!(verified.body.contains("now verified"));
19662        assert_ne!(
19663            f.runner.state.change_summary.as_ref().unwrap().evidence,
19664            original
19665        );
19666        let mut round = round_with_notes(1, Some("内部報告"));
19667        round.reviews.push(ReviewRecord {
19668            attempts: 0,
19669            reviewer: 1,
19670            agent: "alpha".into(),
19671            summary: "内部レビュー".into(),
19672            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
19673            vote: None,
19674            failed: None,
19675            duration_ms: 0,
19676        });
19677        round
19678            .fix
19679            .as_mut()
19680            .unwrap()
19681            .rejected
19682            .push(crate::verdict::Rejection {
19683                id: "R1-1-1".into(),
19684                why: "呼び出し元から到達できない".into(),
19685            });
19686        f.runner.state.reviews = vec![round];
19687        // A description omitting the required ID is unusable, even with a title.
19688        let omitted = posted(&mut f).await;
19689        assert!(omitted.body.contains("No description"));
19690        assert!(omitted.body.contains("R1-1-1"));
19691        assert!(!omitted.body.contains("到達"));
19692        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
19693        let described = posted(&mut f).await;
19694        assert!(described.body.contains("unreachable from callers"));
19695        assert!(described.body.contains("R1-1-1"));
19696        assert!(!described.body.contains("到達"));
19697        let evidence = f
19698            .runner
19699            .state
19700            .change_summary
19701            .as_ref()
19702            .unwrap()
19703            .evidence
19704            .clone();
19705        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
19706        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
19707        assert!(posted(&mut f).await.body.contains("rationale updated"));
19708        assert_ne!(
19709            f.runner.state.change_summary.as_ref().unwrap().evidence,
19710            evidence
19711        );
19712        // Old serialized summaries have no evidence fingerprint and must regenerate.
19713        f.runner
19714            .state
19715            .change_summary
19716            .as_mut()
19717            .unwrap()
19718            .evidence
19719            .clear();
19720        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
19721        assert!(posted(&mut f).await.body.contains("current specification"));
19722    }
19723
19724    #[tokio::test]
19725    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
19726        let mut f = summary_fixture();
19727        let m = posted(&mut f).await;
19728        assert_eq!(m.title, "feat(own): add the own file");
19729        assert!(
19730            m.body
19731                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
19732            "{}",
19733            m.body
19734        );
19735        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
19736        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
19737        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19738        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19739        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
19740        let kept = f.runner.state.change_summary.as_ref().unwrap();
19741        assert_eq!(
19742            (kept.head.as_str(), kept.base.as_str()),
19743            (head.as_str(), base.as_str())
19744        );
19745        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
19746    }
19747
19748    #[tokio::test]
19749    async fn review_only_summary_receives_the_configured_github_language() {
19750        let mut f = summary_fixture();
19751        f.runner.state.config.graph.github_language = "ja".into();
19752        let captured = f.repo.join("summary-prompt.txt");
19753        let author = &mut f.runner.state.config.agents[0];
19754        author.command = vec![
19755            "sh".into(),
19756            "-c".into(),
19757            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19758        ];
19759        author.env.insert(
19760            "PROMPT_FILE".into(),
19761            captured.to_string_lossy().into_owned(),
19762        );
19763        std::fs::write(
19764            &f.reply,
19765            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
19766        )
19767        .unwrap();
19768        let m = posted(&mut f).await;
19769        let prompt = std::fs::read_to_string(captured).unwrap();
19770        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
19771        assert!(!prompt.contains("in English"));
19772        assert_eq!(m.title, "feat(own): ファイルを追加");
19773        assert!(m.body.contains("own.txt を追加します。"));
19774    }
19775
19776    #[tokio::test]
19777    async fn review_fixes_come_after_the_whole_change_summary() {
19778        let mut f = summary_fixture();
19779        f.runner.state.reviews = vec![
19780            round_with_notes(1, Some("handled the PATH case")),
19781            round_with_notes(2, None),
19782        ];
19783        let m = posted(&mut f).await;
19784        let summary = m.body.find("## Summary").expect("summary");
19785        let fixes = m.body.find("## Review rounds").expect("review rounds");
19786        assert!(summary < fixes, "{}", m.body);
19787        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
19788        assert!(m.body.contains("## Final verification"), "{}", m.body);
19789    }
19790
19791    #[tokio::test]
19792    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
19793        let mut f = summary_fixture();
19794        let first = posted(&mut f).await;
19795        let first_head = f.runner.state.change_summary.clone().unwrap().head;
19796        // The same head reuses the record: a different reply is not read.
19797        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
19798        let again = posted(&mut f).await;
19799        assert_eq!((again.title, again.body), (first.title, first.body));
19800
19801        let g = |args: &[&str]| {
19802            let out = std::process::Command::new("git")
19803                .args(args)
19804                .current_dir(&f.repo)
19805                .quiet()
19806                .output()
19807                .expect("spawn git");
19808            assert!(out.status.success(), "git {args:?}");
19809        };
19810        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
19811        g(&["add", "-A"]);
19812        g(&["commit", "-q", "-m", "fix(own): handle more"]);
19813        let second = posted(&mut f).await;
19814        let kept = f.runner.state.change_summary.clone().unwrap();
19815        assert_ne!(kept.head, first_head);
19816        assert!(second.body.contains("Not read yet."), "{}", second.body);
19817        assert!(!second.body.contains("so the first summary is read"));
19818        assert_eq!(second.title, "feat(own): other");
19819    }
19820
19821    #[tokio::test]
19822    async fn an_unavailable_summary_is_stated_and_never_invented() {
19823        let mut f = summary_fixture();
19824        std::fs::write(&f.reply, "no title line here").unwrap();
19825        let m = posted(&mut f).await;
19826        assert!(
19827            m.body
19828                .contains("No description of what this change does could be generated"),
19829            "{}",
19830            m.body
19831        );
19832        assert!(m.body.contains("internal reports remain in local records"));
19833        assert!(f.runner.state.change_summary.is_none());
19834        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
19835        // A preview asks nobody and claims nothing.
19836        let winner = f.runner.state.candidates[0].clone();
19837        let preview = f
19838            .runner
19839            .guarded_pr_message(&winner, None, false)
19840            .await
19841            .unwrap()
19842            .unwrap();
19843        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
19844    }
19845
19846    #[test]
19847    fn pr_message_review_single_commit_uses_its_subject() {
19848        let state = review_state(&["feat(nats): per-role user"]);
19849        let m = pr_message(&state, 'A');
19850        assert_eq!(m.title, "feat(nats): per-role user");
19851        assert!(!m.body.contains("Review the work already"), "{}", m.body);
19852        assert!(m.body.contains("## Commits under review"), "{}", m.body);
19853    }
19854
19855    #[test]
19856    fn pr_message_review_multi_commit_takes_the_oldest() {
19857        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
19858        let m = pr_message(&state, 'A');
19859        assert_eq!(m.title, "feat: the change");
19860        for s in ["feat: the change", "fix: typo", "fix: again"] {
19861            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
19862        }
19863    }
19864
19865    #[test]
19866    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
19867        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
19868            let mut state = review_state(&[first, "fix: later fixup"]);
19869            state.candidates[0].branch = "機能/ログイン".to_owned();
19870            let m = pr_message(&state, 'A');
19871            assert!(
19872                m.title.starts_with("chore: land candidate A of run"),
19873                "{}",
19874                m.title
19875            );
19876        }
19877    }
19878
19879    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
19880        BranchFacts {
19881            commits: commits
19882                .iter()
19883                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
19884                .collect(),
19885            stat: stat.to_owned(),
19886        }
19887    }
19888
19889    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
19890        let mut r = review_round(true, 0, 1, 1, true, true);
19891        r.round = round;
19892        r.fix = notes.map(|n| FixRecord {
19893            agent: "fixer".to_owned(),
19894            addressed: Vec::new(),
19895            rejected: Vec::new(),
19896            notes: n.to_owned(),
19897            committed: true,
19898            failed: None,
19899            duration_ms: 0,
19900            continuation: None,
19901        });
19902        r
19903    }
19904
19905    #[test]
19906    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
19907        let state = review_state(&["ignored"]);
19908        let f = facts(
19909            &[
19910                (
19911                    "fix(login): resolve PATH on macOS",
19912                    "Login shells skip rc files.",
19913                ),
19914                ("fix: address review", ""),
19915            ],
19916            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
19917        );
19918        let m = pr_message_with(&state, 'A', Some(&f));
19919        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
19920        assert!(
19921            m.body
19922                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
19923            "{}",
19924            m.body
19925        );
19926        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
19927        assert!(m.body.contains("## Diff stat"), "{}", m.body);
19928        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
19929        for banned in [
19930            "Review the work already",
19931            "no task statement",
19932            "Original task",
19933        ] {
19934            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
19935        }
19936        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
19937    }
19938
19939    #[test]
19940    fn pr_message_review_truncates_a_huge_first_commit_body() {
19941        let state = review_state(&["ignored"]);
19942        let f = facts(
19943            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
19944            "s",
19945        );
19946        let m = pr_message_with(&state, 'A', Some(&f));
19947        assert!(m.body.len() < 30_000, "{}", m.body.len());
19948        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
19949        assert!(
19950            m.body.contains("- ... 1 more commit(s)"),
19951            "{}",
19952            m.body.len()
19953        );
19954        assert!(m.body.ends_with("magi:candidate-a\n"));
19955    }
19956
19957    #[test]
19958    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
19959        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
19960        assert_eq!(m.title, "feat: x");
19961        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
19962        assert!(!m.body.contains("Diff stat"), "{}", m.body);
19963        assert!(!m.body.contains("no task statement"), "{}", m.body);
19964    }
19965
19966    #[test]
19967    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
19968        let mut state = review_state(&["日本語の件名"]);
19969        state.candidates[0].branch = "fix/macos-login-path".to_owned();
19970        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
19971        state.candidates[0].branch = "機能/ログイン".to_owned();
19972        assert!(
19973            pr_message(&state, 'A')
19974                .title
19975                .starts_with("chore: land candidate A")
19976        );
19977    }
19978
19979    #[test]
19980    fn pr_message_review_fixes_survive_a_clean_final_round() {
19981        let mut state = review_state(&["feat: x"]);
19982        state.reviews = vec![
19983            round_with_notes(1, Some("handled the PATH case")),
19984            round_with_notes(2, None),
19985        ];
19986        let body = pr_message(&state, 'A').body;
19987        assert!(
19988            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
19989            "{body}"
19990        );
19991        assert!(body.contains("### Round 1"), "{body}");
19992
19993        state.reviews = vec![
19994            round_with_notes(1, Some("first fix")),
19995            round_with_notes(2, Some("")),
19996            round_with_notes(3, Some("second fix")),
19997            round_with_notes(4, None),
19998        ];
19999        let body = pr_message(&state, 'A').body;
20000        assert!(
20001            body.contains("### Round 1") && !body.contains("first fix"),
20002            "{body}"
20003        );
20004        assert!(
20005            body.contains("### Round 3") && !body.contains("second fix"),
20006            "{body}"
20007        );
20008        assert!(body.contains("### Round 2"), "{body}");
20009    }
20010
20011    const FINAL: &str = "abcdef1234567890";
20012
20013    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
20014        let mut r = round_with_notes(round, None);
20015        r.verified_head = verified.map(str::to_owned);
20016        r
20017    }
20018
20019    fn ok_gate(state: &mut RunState, head: Option<&str>) {
20020        state.gate = vec![CommandOutcome {
20021            command: "gate".to_owned(),
20022            code: Some(0),
20023            output_tail: String::new(),
20024            duration_ms: 0,
20025            resource_blocked: false,
20026        }];
20027        state.gate_ran = true;
20028        state.gate_head = head.map(str::to_owned);
20029    }
20030
20031    fn body_at(state: &RunState) -> String {
20032        pr_message_raw(
20033            state,
20034            'A',
20035            None,
20036            crate::pr_outcome::FinalHead::At(FINAL),
20037            ChangeNote::NotAsked,
20038        )
20039        .body
20040    }
20041
20042    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
20043
20044    #[test]
20045    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
20046        let mut state = review_state(&["feat: x"]);
20047        state.reviews = vec![
20048            round_with_notes(1, Some(SANDBOXED)),
20049            passing_round(2, Some(FINAL)),
20050        ];
20051        ok_gate(&mut state, Some(FINAL));
20052        let body = body_at(&state);
20053        assert!(!body.contains("Handled the PATH case."), "{body}");
20054        assert!(!body.contains("could not commit"), "{body}");
20055        assert!(!body.contains("could not run"), "{body}");
20056        assert!(
20057            body.contains("The fix is committed on the branch"),
20058            "{body}"
20059        );
20060        assert!(!body.contains("sandbox"), "{body}");
20061        assert!(
20062            body.contains("Tests (`verify.e2e`): passed on the final commit"),
20063            "{body}"
20064        );
20065        assert!(
20066            body.contains("Gate (`verify.gate`): passed on the final commit"),
20067            "{body}"
20068        );
20069        // The raw report is untouched in the record.
20070        assert!(
20071            state.reviews[0]
20072                .fix
20073                .as_ref()
20074                .unwrap()
20075                .notes
20076                .contains("could not commit")
20077        );
20078    }
20079
20080    #[test]
20081    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
20082        let mut state = review_state(&["feat: x"]);
20083        let mut r1 = round_with_notes(1, Some(SANDBOXED));
20084        r1.fix.as_mut().unwrap().committed = false;
20085        let mut r2 = passing_round(2, Some(FINAL));
20086        r2.e2e[0].code = Some(1);
20087        state.reviews = vec![r1, r2];
20088        let body = body_at(&state);
20089        assert!(!body.contains("could not commit"), "{body}");
20090        assert!(!body.contains("could not run"), "{body}");
20091        assert!(body.contains("This round produced no commit"), "{body}");
20092        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
20093        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
20094        assert!(!body.contains("passed"), "{body}");
20095    }
20096
20097    #[test]
20098    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
20099        let mut state = review_state(&["feat: x"]);
20100        state.reviews = vec![
20101            round_with_notes(1, Some(SANDBOXED)),
20102            passing_round(2, Some("0123456789abcdef")),
20103        ];
20104        ok_gate(&mut state, Some("0123456789abcdef"));
20105        let body = body_at(&state);
20106        assert!(!body.contains("HTTP tests could not run"), "{body}");
20107        // The commit claim is still settled by the structured record.
20108        assert!(!body.contains("could not commit"), "{body}");
20109        assert!(
20110            body.contains("passed on an earlier commit (`0123456`)"),
20111            "{body}"
20112        );
20113        assert!(!body.contains("passed on the final commit"), "{body}");
20114
20115        // A gate recorded before `gate_head` existed is "not recorded".
20116        ok_gate(&mut state, None);
20117        let body = body_at(&state);
20118        assert!(
20119            body.contains("passed on a commit that was not recorded"),
20120            "{body}"
20121        );
20122    }
20123
20124    #[test]
20125    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
20126        let state = state_with_summary(
20127            "add retries to the client",
20128            "TITLE: feat: retries\n\nDid it.",
20129        );
20130        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
20131        assert_eq!(m.title, "feat: retries");
20132        assert!(
20133            m.body.contains("<summary>Original task</summary>"),
20134            "{}",
20135            m.body
20136        );
20137        assert!(!m.body.contains("Commits under review"), "{}", m.body);
20138        assert!(
20139            m.body
20140                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
20141            "{}",
20142            m.body
20143        );
20144    }
20145
20146    #[test]
20147    fn pr_message_review_bounds_a_long_english_subject() {
20148        let long = format!("feat: {}", "word ".repeat(100));
20149        let m = pr_message(&review_state(&[&long]), 'A');
20150        assert!(m.title.starts_with("feat: word"), "{}", m.title);
20151        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20152    }
20153
20154    #[test]
20155    fn pr_message_implementation_run_is_unchanged_by_review_support() {
20156        let state = state_with_summary("add retries\n\ndetails", "- did some things");
20157        let m = pr_message(&state, 'A');
20158        assert_eq!(m.title, "add retries");
20159        assert!(m.body.contains("<summary>Original task</summary>"));
20160        assert!(!m.body.contains("Commits under review"));
20161        assert_eq!(landing_subject_source(&state), state.instruction);
20162    }
20163
20164    #[test]
20165    fn review_run_squash_subject_is_the_change_not_the_prompt() {
20166        let state = review_state(&["feat: the change", "fix: typo"]);
20167        let source = landing_subject_source(&state);
20168        assert_eq!(land::merge_subject("", &source), "feat: the change");
20169        assert_eq!(
20170            land::merge_subject("magi: candidate A (uncommitted work)", &source),
20171            "feat: the change"
20172        );
20173        // An operator's rename still wins.
20174        assert_eq!(
20175            land::merge_subject("feat: renamed by hand", &source),
20176            "feat: renamed by hand"
20177        );
20178        let blank = review_state(&["日本語"]);
20179        assert!(
20180            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
20181        );
20182    }
20183
20184    #[test]
20185    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
20186        let state = review_state(&["feat: the change"]);
20187        let source = landing_subject_source(&state);
20188        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
20189        assert_eq!(
20190            land::merge_subject(landing_title(&state, old), &source),
20191            "feat: the change"
20192        );
20193        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
20194        let task = state_with_summary("add retries", "");
20195        assert_eq!(landing_title(&task, old), old);
20196    }
20197
20198    #[test]
20199    fn pr_message_describes_the_change_not_the_task() {
20200        let state = state_with_summary(
20201            "今回やってほしいこと: results projector を直す",
20202            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
20203        );
20204        let m = pr_message(&state, 'A');
20205        assert_eq!(m.title, "fix(web): batch the runs list reads");
20206        assert!(
20207            m.body.starts_with("## Summary\n\n- reads run.json once"),
20208            "{}",
20209            m.body
20210        );
20211        assert!(!m.body.contains("TITLE:"), "{}", m.body);
20212        let task_at = m.body.find("今回やってほしいこと").unwrap();
20213        let details_at = m.body.find("<details>").unwrap();
20214        assert!(
20215            details_at < task_at,
20216            "the task lives inside <details>: {}",
20217            m.body
20218        );
20219        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
20220        assert!(m.body.contains("magi:candidate-a"));
20221    }
20222
20223    #[test]
20224    fn pr_message_falls_back_to_the_task_without_a_title_line() {
20225        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
20226        let m = pr_message(&state, 'A');
20227        assert_eq!(m.title, "add retries");
20228        assert!(
20229            m.body.contains("## Summary\n\n- did some things"),
20230            "{}",
20231            m.body
20232        );
20233
20234        let none = RunState::new(
20235            PathBuf::from("/repo"),
20236            "main".to_owned(),
20237            "abc1234".to_owned(),
20238            "add retries".to_owned(),
20239            Config::default(),
20240        );
20241        let m = pr_message(&none, 'A');
20242        assert_eq!(m.title, "add retries");
20243        assert!(!m.body.contains("## Summary"), "{}", m.body);
20244    }
20245
20246    #[test]
20247    fn pr_message_refuses_the_candidate_commit_subject() {
20248        for bad in [
20249            "TITLE: magi: candidate A (uncommitted work)",
20250            "TITLE: chore: stuff (uncommitted work)",
20251            "TITLE:   ",
20252        ] {
20253            let state = state_with_summary("add retries", bad);
20254            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
20255        }
20256    }
20257
20258    #[test]
20259    fn pr_message_bounds_a_very_long_task_and_title() {
20260        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
20261        let state = state_with_summary(&long, "- nothing");
20262        let m = pr_message(&state, 'A');
20263        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20264        assert!(!m.title.contains('\n'));
20265
20266        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
20267        let m = pr_message(&state, 'A');
20268        assert!(m.title.starts_with("feat: "));
20269        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20270        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
20271    }
20272
20273    fn long_title_of(instruction: &str) -> String {
20274        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
20275    }
20276
20277    #[test]
20278    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
20279        let first = "Make the landing path keep a readable title for long tasks";
20280        let line = format!(
20281            "{first}. {}",
20282            "And then keep going with more words ".repeat(20)
20283        );
20284        let t = long_title_of(&line);
20285        assert_eq!(t, first);
20286        assert!(!t.starts_with("chore: land"));
20287    }
20288
20289    #[test]
20290    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
20291        let line = "word ".repeat(200);
20292        let t = long_title_of(&line);
20293        assert!(t.ends_with("word..."), "{t}");
20294        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
20295    }
20296
20297    #[test]
20298    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
20299        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
20300            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
20301        }
20302    }
20303
20304    #[test]
20305    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
20306        let mut state = state_with_summary("再試行を追加する", "- nothing");
20307        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
20308        state.config.graph.github_language = "ja".into();
20309        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
20310    }
20311
20312    #[test]
20313    fn pr_message_title_limit_is_exact() {
20314        let at = "a".repeat(PR_TITLE_MAX);
20315        assert_eq!(long_title_of(&at), at);
20316        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
20317        assert!(over.ends_with("..."), "{over}");
20318        assert_eq!(over.chars().count(), PR_TITLE_MAX);
20319    }
20320
20321    #[test]
20322    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
20323        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
20324        let t = long_title_of(&line);
20325        assert!(t.ends_with("..."), "{t}");
20326        assert!(t.is_ascii(), "{t}");
20327    }
20328
20329    #[test]
20330    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
20331        let line = format!(
20332            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
20333            "plus more filler words ".repeat(20)
20334        );
20335        let t = long_title_of(&line);
20336        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
20337    }
20338
20339    #[test]
20340    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
20341        let line = format!("\n\n# {}", "title words ".repeat(40));
20342        let state = state_with_summary(&line, "- nothing");
20343        let m = pr_message(&state, 'A');
20344        assert!(m.title.starts_with("title words"), "{}", m.title);
20345        assert_eq!(
20346            land::merge_subject(&m.title, &landing_subject_source(&state)),
20347            m.title
20348        );
20349        // An operator's rename wins untouched.
20350        assert_eq!(
20351            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
20352            "feat: renamed by hand"
20353        );
20354    }
20355
20356    #[tokio::test]
20357    async fn github_text_rewrite_is_bounded_and_falls_back() {
20358        crate::run::pin_test_home();
20359        for (reply, accepted) in [
20360            (
20361                "TITLE: fix: retries\nAdd retries for failed requests.",
20362                true,
20363            ),
20364            (
20365                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
20366                false,
20367            ),
20368            (
20369                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
20370                true,
20371            ),
20372            (
20373                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
20374                true,
20375            ),
20376            (
20377                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
20378                true,
20379            ),
20380            (
20381                "TITLE: fix: retries\n日本語の説明です。 token=secret",
20382                false,
20383            ),
20384            ("TITLE: fix: retries\n", false),
20385        ] {
20386            let dir = tempfile::tempdir().unwrap();
20387            let mut runner = runner_at(RunStatus::Gating);
20388            runner.state = state_with_summary(
20389                "add retries",
20390                "TITLE: fix: retries\n日本語の説明を書きます。",
20391            );
20392            runner.state.repo = dir.path().to_owned();
20393            runner.state.candidates[0].worktree = dir.path().to_owned();
20394            let mut author = spec("alpha");
20395            author.command = vec![
20396                "sh".into(),
20397                "-c".into(),
20398                "printf '%s' \"$REWRITE_REPLY\"".into(),
20399            ];
20400            author.env.insert("REWRITE_REPLY".into(), reply.into());
20401            runner.state.config.agents = vec![author];
20402            seed_public_summary(&mut runner);
20403            let winner = runner.state.candidates[0].clone();
20404            let first = runner
20405                .guarded_pr_message(&winner, None, true)
20406                .await
20407                .unwrap();
20408            let message = if accepted {
20409                first.expect("a passing rewrite never asks")
20410            } else {
20411                // Withheld: the run parks on a question and posts nothing yet.
20412                assert!(first.is_none());
20413                assert!(runner.state.parked);
20414                let store = ask::Questions::open();
20415                let asked: Vec<_> = store
20416                    .list()
20417                    .into_iter()
20418                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20419                    .collect();
20420                assert_eq!(asked.len(), 1);
20421                assert!(!asked[0].detail.contains("secret"));
20422                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
20423                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
20424                assert_eq!(
20425                    asked[0]
20426                        .choices
20427                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
20428                    rewrite.is_some()
20429                );
20430                // Silence: the question lapses and the neutral text is used,
20431                // without a second rewrite call or a second question.
20432                store
20433                    .update(&asked[0].id, |q| {
20434                        q.abandon("test");
20435                        Ok(())
20436                    })
20437                    .unwrap();
20438                let again = runner
20439                    .guarded_pr_message(&winner, None, true)
20440                    .await
20441                    .unwrap()
20442                    .expect("an abandoned question falls back");
20443                let n = store
20444                    .list()
20445                    .into_iter()
20446                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20447                    .count();
20448                assert_eq!(n, 1, "asked once per rejected text");
20449                again
20450            };
20451            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
20452            assert!(!message.body.contains("secret"));
20453            assert!(!message.body.contains("prefix"));
20454            assert!(!message.title.contains("secret"));
20455            assert_eq!(
20456                message.body.contains("Add retries for failed requests."),
20457                accepted
20458            );
20459            assert_eq!(
20460                runner.state.seats["github-text-rewrite"].turns, 1,
20461                "only one rewrite invocation"
20462            );
20463            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
20464            if accepted {
20465                assert!(runner.state.github_text.is_none());
20466                assert!(!runner.state.parked);
20467                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
20468                assert!(message.body.contains("## Final verification"));
20469                assert!(message.body.contains("Evidence: final diff"));
20470                assert!(
20471                    message
20472                        .body
20473                        .contains(&format!("magi:run/{}", runner.state.id))
20474                );
20475            }
20476            if !accepted {
20477                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20478                assert!(
20479                    message
20480                        .body
20481                        .contains(&format!("magi:run/{}", runner.state.id))
20482                );
20483            }
20484        }
20485    }
20486
20487    fn scripted(id: &str, reply: &str) -> AgentSpec {
20488        let mut a = spec(id);
20489        a.command = vec![
20490            "sh".into(),
20491            "-c".into(),
20492            "printf '%s' \"$REWRITE_REPLY\"".into(),
20493        ];
20494        a.env.insert("REWRITE_REPLY".into(), reply.into());
20495        a
20496    }
20497
20498    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
20499    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
20500
20501    /// Gate tests inject generated public output, not internal reports.
20502    fn seed_public_summary(runner: &mut Runner) {
20503        let repo = &runner.state.repo;
20504        let g = |args: &[&str]| {
20505            let out = std::process::Command::new("git")
20506                .args(args)
20507                .current_dir(repo)
20508                .quiet()
20509                .output()
20510                .unwrap();
20511            assert!(out.status.success(), "git {args:?}");
20512            String::from_utf8_lossy(&out.stdout).trim().to_owned()
20513        };
20514        g(&["init", "-q", "-b", "main"]);
20515        g(&[
20516            "-c",
20517            "user.name=Test",
20518            "-c",
20519            "user.email=test@example.invalid",
20520            "commit",
20521            "-q",
20522            "--allow-empty",
20523            "-m",
20524            "feat: fixture",
20525        ]);
20526        let head = g(&["rev-parse", "HEAD"]);
20527        g(&["update-ref", "refs/remotes/origin/main", &head]);
20528        let evidence = crate::pr_outcome::summary_evidence(
20529            &runner.state,
20530            crate::pr_outcome::FinalHead::At(&head),
20531        );
20532        let generated = &runner.state.candidates[0].summary;
20533        runner.state.change_summary = Some(crate::run::ChangeSummary {
20534            base: head.clone(),
20535            head,
20536            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
20537            title: summary_title(generated).unwrap(),
20538            body: summary_without_title(generated),
20539        });
20540    }
20541
20542    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
20543        let mut runner = runner_at(RunStatus::Gating);
20544        runner.state = state_with_summary("add retries", JA_SUMMARY);
20545        if review_only {
20546            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
20547        }
20548        runner.state.repo = dir.to_owned();
20549        runner.state.candidates[0].worktree = dir.to_owned();
20550        runner.state.config.agents = agents;
20551        seed_public_summary(&mut runner);
20552        runner
20553    }
20554
20555    fn gate_questions(runner: &Runner) -> usize {
20556        ask::Questions::open()
20557            .list()
20558            .into_iter()
20559            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20560            .count()
20561    }
20562
20563    #[tokio::test]
20564    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
20565        crate::run::pin_test_home();
20566        let dir = tempfile::tempdir().unwrap();
20567        let mut runner = rewrite_runner(
20568            dir.path(),
20569            true,
20570            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20571        );
20572        runner.state.config.roles.language_judge =
20573            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
20574        let winner = runner.state.candidates[0].clone();
20575        let msg = runner
20576            .guarded_pr_message(&winner, None, true)
20577            .await
20578            .unwrap()
20579            .expect("an English rewrite posts without asking");
20580        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
20581        assert!(msg.body.contains("<details>kept</details>"));
20582        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
20583        assert_eq!(gate_questions(&runner), 0);
20584        assert!(runner.state.github_text.is_none());
20585        assert!(!runner.state.seats.contains_key("impl-A"));
20586        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20587    }
20588
20589    #[tokio::test]
20590    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
20591        crate::run::pin_test_home();
20592        let dir = tempfile::tempdir().unwrap();
20593        let mut runner = rewrite_runner(
20594            dir.path(),
20595            false,
20596            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20597        );
20598        runner.state.config.roles.language_judge =
20599            Some(crate::config::AgentChoice::One("beta".into()));
20600        runner.state.seats.insert(
20601            "impl-A".into(),
20602            SeatState::new("impl-A", "alpha", runner.state.seed),
20603        );
20604        let winner = runner.state.candidates[0].clone();
20605        let msg = runner
20606            .guarded_pr_message(&winner, None, true)
20607            .await
20608            .unwrap()
20609            .expect("rewritten");
20610        assert!(msg.body.contains("<details>kept</details>"));
20611        assert_eq!(runner.state.seats["impl-A"].turns, 0);
20612        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20613        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20614    }
20615
20616    #[tokio::test]
20617    async fn rewrite_reuses_a_resumable_implementer_session() {
20618        crate::run::pin_test_home();
20619        let dir = tempfile::tempdir().unwrap();
20620        let mut runner = rewrite_runner(
20621            dir.path(),
20622            false,
20623            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
20624        );
20625        runner.state.config.roles.language_judge =
20626            Some(crate::config::AgentChoice::One("beta".into()));
20627        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20628        seat.turns = 1;
20629        runner.state.seats.insert("impl-A".into(), seat);
20630        let winner = runner.state.candidates[0].clone();
20631        runner
20632            .guarded_pr_message(&winner, None, true)
20633            .await
20634            .unwrap()
20635            .expect("rewritten");
20636        assert_eq!(runner.state.seats["impl-A"].turns, 2);
20637        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
20638    }
20639
20640    #[tokio::test]
20641    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
20642        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
20643        // session, was out of quota, and nothing else was asked.
20644        crate::run::pin_test_home();
20645        let dir = tempfile::tempdir().unwrap();
20646        let mut exhausted = spec("alpha");
20647        exhausted.command = vec!["false".into()];
20648        let mut runner = rewrite_runner(
20649            dir.path(),
20650            false,
20651            vec![exhausted, scripted("beta", EN_REWRITE)],
20652        );
20653        // The implementer's own agent is in the role too: it is not asked twice.
20654        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20655            "alpha".into(),
20656            "beta".into(),
20657        ]));
20658        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20659        seat.turns = 1;
20660        runner.state.seats.insert("impl-A".into(), seat);
20661        let winner = runner.state.candidates[0].clone();
20662        let msg = runner
20663            .guarded_pr_message(&winner, None, true)
20664            .await
20665            .unwrap()
20666            .expect("the role rewrites after the implementer fails");
20667        assert!(msg.body.contains("`src/net.rs`"));
20668        assert_eq!(
20669            runner.state.seats["impl-A"].turns, 2,
20670            "the implementer was tried once"
20671        );
20672        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20673        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20674    }
20675
20676    #[tokio::test]
20677    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
20678        crate::run::pin_test_home();
20679        let dir = tempfile::tempdir().unwrap();
20680        let mut broken = spec("bad");
20681        broken.command = vec!["false".into()];
20682        let mut runner = rewrite_runner(
20683            dir.path(),
20684            true,
20685            vec![
20686                broken,
20687                scripted("noformat", "Add retries."),
20688                scripted("good", EN_REWRITE),
20689            ],
20690        );
20691        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20692            "bad".into(),
20693            "noformat".into(),
20694            "good".into(),
20695        ]));
20696        let winner = runner.state.candidates[0].clone();
20697        let msg = runner
20698            .guarded_pr_message(&winner, None, true)
20699            .await
20700            .unwrap()
20701            .expect("the third agent rewrites");
20702        assert!(msg.body.contains("`src/net.rs`"));
20703        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
20704        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
20705    }
20706
20707    #[tokio::test]
20708    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
20709        crate::run::pin_test_home();
20710        // (agents, role): still Japanese; no TITLE; nothing resolvable.
20711        for (agents, role, saved) in [
20712            (
20713                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
20714                Some("beta"),
20715                true,
20716            ),
20717            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
20718            (Vec::new(), None, false),
20719        ] {
20720            let dir = tempfile::tempdir().unwrap();
20721            let mut runner = rewrite_runner(dir.path(), true, agents);
20722            runner.state.config.roles.language_judge =
20723                role.map(|r| crate::config::AgentChoice::One(r.into()));
20724            let winner = runner.state.candidates[0].clone();
20725            let first = runner
20726                .guarded_pr_message(&winner, None, true)
20727                .await
20728                .unwrap();
20729            assert!(first.is_none(), "withheld text parks the run");
20730            assert!(runner.state.parked);
20731            assert_eq!(gate_questions(&runner), 1);
20732            let g = runner.state.github_text.as_ref().unwrap();
20733            assert_eq!(g.rewrite.is_some(), saved);
20734            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
20735        }
20736    }
20737
20738    #[tokio::test]
20739    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
20740        crate::run::pin_test_home();
20741        let dir = tempfile::tempdir().unwrap();
20742        let mut runner = runner_at(RunStatus::Gating);
20743        runner.state =
20744            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
20745        runner.state.repo = dir.path().to_owned();
20746        runner.state.candidates[0].worktree = dir.path().to_owned();
20747        let mut author = spec("alpha");
20748        author.command = vec![
20749            "sh".into(),
20750            "-c".into(),
20751            "printf '%s' \"$REWRITE_REPLY\"".into(),
20752        ];
20753        author.env.insert(
20754            "REWRITE_REPLY".into(),
20755            format!(
20756                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
20757                "English sentence. ".repeat(600)
20758            ),
20759        );
20760        runner.state.config.agents = vec![author];
20761        seed_public_summary(&mut runner);
20762        let winner = runner.state.candidates[0].clone();
20763        assert!(
20764            runner
20765                .guarded_pr_message(&winner, None, true)
20766                .await
20767                .unwrap()
20768                .is_none()
20769        );
20770        let record = runner.state.github_text.clone().unwrap();
20771        let rewrite = record.rewrite.clone().unwrap();
20772        assert_eq!(rewrite.failed, ["body-language"]);
20773        assert!(
20774            rewrite
20775                .body
20776                .ends_with(&format!("magi:run/{}", runner.state.id))
20777        );
20778        let store = ask::Questions::open();
20779        let q = store.get(record.question.as_ref().unwrap()).unwrap();
20780        assert!(
20781            q.choices
20782                .contains(&crate::github_text::USE_REWRITE.to_owned())
20783        );
20784        for label in [
20785            "ORIGINAL",
20786            "FALLBACK",
20787            "REWRITE",
20788            "Rules still failed: body-language",
20789            "truncated",
20790        ] {
20791            assert!(q.detail.contains(label), "{label}");
20792        }
20793        let artifact = std::fs::read_to_string(crate::run::artifact_path(
20794            &runner.state,
20795            &crate::github_text::artifact_name(&record.fingerprint),
20796        ))
20797        .unwrap();
20798        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
20799        // Restart from serialized state, then retry after a bad owner title.
20800        runner.state =
20801            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
20802        store
20803            .update(&q.id, |q| {
20804                q.say("まだ日本語")?;
20805                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
20806            })
20807            .unwrap();
20808        assert!(
20809            runner
20810                .guarded_pr_message(&winner, None, true)
20811                .await
20812                .unwrap()
20813                .is_none()
20814        );
20815        let retry = runner.state.github_text.as_ref().unwrap();
20816        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
20817        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
20818        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
20819        assert!(
20820            q.posting_comparison
20821                .as_ref()
20822                .unwrap()
20823                .matches_detail(&q.detail)
20824        );
20825        store
20826            .update(&q.id, |q| {
20827                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20828            })
20829            .unwrap();
20830        let message = runner
20831            .guarded_pr_message(&winner, None, true)
20832            .await
20833            .unwrap()
20834            .unwrap();
20835        assert_eq!(message.title, rewrite.title);
20836        assert_eq!(message.body, rewrite.body);
20837        assert_eq!(
20838            runner.state.github_text.as_ref().unwrap().choice,
20839            Some(GithubTextChoice::Rewrite)
20840        );
20841        let again = runner
20842            .guarded_pr_message(&winner, None, true)
20843            .await
20844            .unwrap()
20845            .unwrap();
20846        assert_eq!(again.title, message.title);
20847        assert_eq!(again.body, message.body);
20848        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20849        assert!(
20850            !runner
20851                .state
20852                .events
20853                .iter()
20854                .any(|e| e.message.contains(&rewrite.title)
20855                    || e.message.contains("English sentence."))
20856        );
20857    }
20858
20859    #[tokio::test]
20860    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
20861        crate::run::pin_test_home();
20862        let dir = tempfile::tempdir().unwrap();
20863        let mut runner = runner_at(RunStatus::Gating);
20864        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
20865        runner.state.repo = dir.path().to_owned();
20866        runner.state.candidates[0].worktree = dir.path().to_owned();
20867        seed_public_summary(&mut runner);
20868        let winner = runner.state.candidates[0].clone();
20869        assert!(
20870            runner
20871                .guarded_pr_message(&winner, None, true)
20872                .await
20873                .unwrap()
20874                .is_none()
20875        );
20876        let record = runner.state.github_text.as_ref().unwrap();
20877        assert!(record.rewrite.is_none());
20878        // Records written before the additive field deserialize with no rewrite.
20879        let mut old = serde_json::to_value(record).unwrap();
20880        old.as_object_mut().unwrap().remove("rewrite");
20881        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
20882        assert!(old.rewrite.is_none());
20883        let id = record.question.clone().unwrap();
20884        ask::Questions::open()
20885            .update(&id, |q| {
20886                q.choices.push(crate::github_text::USE_REWRITE.into());
20887                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20888            })
20889            .unwrap();
20890        let message = runner
20891            .guarded_pr_message(&winner, None, true)
20892            .await
20893            .unwrap()
20894            .unwrap();
20895        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20896        assert_eq!(
20897            runner.state.github_text.as_ref().unwrap().choice,
20898            Some(GithubTextChoice::Fallback)
20899        );
20900    }
20901
20902    #[tokio::test]
20903    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
20904        crate::run::pin_test_home();
20905        let dir = tempfile::tempdir().unwrap();
20906        let mut runner = runner_at(RunStatus::Gating);
20907        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
20908        runner.state.repo = dir.path().to_owned();
20909        runner.state.candidates[0].worktree = dir.path().to_owned();
20910        seed_public_summary(&mut runner);
20911        let winner = runner.state.candidates[0].clone();
20912        assert!(
20913            runner
20914                .guarded_pr_message(&winner, None, true)
20915                .await
20916                .unwrap()
20917                .is_none()
20918        );
20919        let store = ask::Questions::open();
20920        let find = |run: &str| {
20921            store
20922                .list()
20923                .into_iter()
20924                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
20925                .unwrap()
20926        };
20927        let run = runner.state.id.clone();
20928        let q = find(&run);
20929        let fp = runner
20930            .state
20931            .github_text
20932            .as_ref()
20933            .unwrap()
20934            .fingerprint
20935            .clone();
20936        let name = crate::github_text::artifact_name(&fp);
20937        let path = crate::run::artifact_path(&runner.state, &name);
20938        assert!(q.detail.contains(&path.display().to_string()));
20939        assert!(q.detail.contains("title-language: title"));
20940        let saved = std::fs::read_to_string(&path).expect("artifact written");
20941        assert!(saved.contains("日本語のタイトル"));
20942        assert!(q.cwd.is_none());
20943        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
20944        // A bad replacement is refused and asked about once more.
20945        let say = |id: &str, text: &str, choice: &str| {
20946            store
20947                .update(id, |q| {
20948                    q.thread.push(ask::Turn {
20949                        who: ask::Who::Operator,
20950                        body: text.to_owned(),
20951                        at: jiff::Timestamp::now(),
20952                        note: None,
20953                    });
20954                    q.answer(ask::Answer::Choice(choice.to_owned()))
20955                })
20956                .unwrap();
20957        };
20958        say(&q.id, "まだ日本語", "use my text");
20959        assert!(
20960            runner
20961                .guarded_pr_message(&winner, None, true)
20962                .await
20963                .unwrap()
20964                .is_none()
20965        );
20966        let second = find(&run);
20967        assert_ne!(second.id, q.id);
20968        assert!(second.detail.contains("did not pass"));
20969        // The retry still compares the original text; the rejected
20970        // replacement is named by category only.
20971        assert!(second.detail.contains("日本語のタイトル"));
20972        assert!(!second.detail.contains("まだ日本語"));
20973        assert!(second.detail.contains(&path.display().to_string()));
20974        say(&second.id, "fix: retry failed requests", "use my text");
20975        let message = runner
20976            .guarded_pr_message(&winner, None, true)
20977            .await
20978            .unwrap()
20979            .expect("a vetted replacement is posted");
20980        assert_eq!(message.title, "fix: retry failed requests");
20981        assert_eq!(
20982            runner
20983                .state
20984                .github_text
20985                .as_ref()
20986                .unwrap()
20987                .chosen_title
20988                .as_deref(),
20989            Some("fix: retry failed requests")
20990        );
20991        assert!(runner.state.github_text.as_ref().unwrap().resolved);
20992        // Resumed again: the saved decision is applied, nothing is re-asked.
20993        let again = runner
20994            .guarded_pr_message(&winner, None, true)
20995            .await
20996            .unwrap()
20997            .unwrap();
20998        assert_eq!(again.title, message.title);
20999        assert_eq!(
21000            store
21001                .list()
21002                .into_iter()
21003                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
21004                .count(),
21005            2
21006        );
21007    }
21008
21009    #[test]
21010    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
21011        // What magi itself writes stays English under any configured language,
21012        // so a future localisation of these headings fails here. (The agents'
21013        // own text is also checked by the posting gate.)
21014        let mut state = state_with_summary(
21015            "add retries",
21016            "TITLE: fix(web): batch reads\n- reads run.json once",
21017        );
21018        state.config.graph.language = "ja".to_owned();
21019        let m = pr_message(&state, 'A');
21020        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
21021        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
21022
21023        // The task is the operator's own text: it goes in untouched, and the
21024        // fallback title (no summary) may be in its language too.
21025        let task = "今回やってほしいこと: results projector を直す";
21026        let mut state = state_with_summary(task, "- no title line");
21027        state.config.graph.language = "ja".to_owned();
21028        let m = pr_message(&state, 'A');
21029        assert_eq!(
21030            m.title,
21031            format!("chore: land candidate A of run {}", state.id)
21032        );
21033        assert!(
21034            m.body.contains(&format!(
21035                "<summary>Original task</summary>\n\n{task}\n\n</details>"
21036            )),
21037            "{}",
21038            m.body
21039        );
21040    }
21041
21042    #[test]
21043    fn pr_message_scrubs_home_paths_and_addresses() {
21044        let state = state_with_summary(
21045            "fix it in /Users/someone/src/x",
21046            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
21047        );
21048        let m = pr_message(&state, 'A');
21049        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
21050            assert!(!m.body.contains(leak), "{}", m.body);
21051        }
21052        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
21053    }
21054
21055    #[test]
21056    fn pr_message_survives_a_task_that_closes_details() {
21057        let state = state_with_summary("a </details> b", "TITLE: fix: x");
21058        let m = pr_message(&state, 'A');
21059        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
21060    }
21061
21062    #[test]
21063    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
21064        let cmd = manual_merge_command(
21065            MergeStyle::Squash,
21066            Path::new("/repo"),
21067            "b",
21068            "fix: \"quoted\" $(x) `y`\n\nbody",
21069        );
21070        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
21071    }
21072
21073    #[test]
21074    fn manual_merge_command_matches_the_configured_style() {
21075        let repo = Path::new("/repo");
21076        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
21077
21078        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
21079        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
21080
21081        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
21082        assert_eq!(
21083            squash,
21084            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
21085             \"Merge magi run 0832 (candidate A)\""
21086        );
21087
21088        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
21089        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
21090    }
21091
21092    #[test]
21093    fn a_nudge_gets_a_quarter_of_the_budget() {
21094        // The judge and implement budgets magi ships with.
21095        assert_eq!(retry_budget(secs(1200), true), secs(300));
21096        assert_eq!(retry_budget(secs(3600), true), secs(900));
21097    }
21098
21099    #[test]
21100    fn a_resent_prompt_keeps_the_whole_budget() {
21101        // The seat kept no context, so the retry is the original job again and
21102        // shortening it would only guarantee a second failure.
21103        assert_eq!(retry_budget(secs(1200), false), secs(1200));
21104        assert_eq!(retry_budget(secs(60), false), secs(60));
21105    }
21106
21107    #[test]
21108    fn the_floor_never_exceeds_the_original_budget() {
21109        // A short configured timeout must not be *raised* by the floor: the
21110        // operator asked for a bound, and a retry may not outlast the attempt
21111        // it is retrying.
21112        assert_eq!(retry_budget(secs(60), true), secs(60));
21113        assert_eq!(retry_budget(secs(480), true), secs(120));
21114        assert_eq!(retry_budget(secs(0), true), secs(0));
21115    }
21116
21117    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
21118        agent::CommandEvidence {
21119            id: "item1".to_owned(),
21120            description: "cargo test".to_owned(),
21121            exit_code,
21122            result_summary: String::new(),
21123            source: "codex".to_owned(),
21124        }
21125    }
21126
21127    #[test]
21128    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
21129        // No evidence is not the same fact as unconfirmed evidence: a
21130        // backend with no adapter, or a reply that ran no commands at all,
21131        // must not be misread as carrying a dangling job.
21132        assert!(!has_unconfirmed_command(&[]));
21133    }
21134
21135    #[test]
21136    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
21137        // Deliberately not a check on the exit code's *value*: a fixer
21138        // legitimately runs something that fails mid-iteration before it
21139        // succeeds, and that must never by itself reopen a valid report.
21140        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
21141        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
21142        assert!(!has_unconfirmed_command(&[
21143            evidence(Some(0)),
21144            evidence(Some(101))
21145        ]));
21146    }
21147
21148    #[test]
21149    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
21150        assert!(has_unconfirmed_command(&[
21151            evidence(Some(0)),
21152            evidence(None)
21153        ]));
21154    }
21155
21156    #[test]
21157    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
21158        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21159        assert_eq!(
21160            verified_noop_claim(true, &[], text).as_deref(),
21161            Some("already fixed by b32cfc4, on main.")
21162        );
21163    }
21164
21165    #[test]
21166    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
21167        // A timeout or a bad exit code reads as the ordinary loss it is,
21168        // whatever the reply's own prose claims.
21169        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21170        assert!(verified_noop_claim(false, &[], text).is_none());
21171    }
21172
21173    #[test]
21174    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
21175        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21176        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
21177        // A confirmed command alongside the marker is fine.
21178        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
21179    }
21180
21181    #[test]
21182    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
21183        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
21184    }
21185
21186    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
21187    /// pair, in order, labelled A, B, C, ...
21188    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
21189        runner.state.candidates = shape
21190            .iter()
21191            .enumerate()
21192            .map(|(i, &(empty, verified))| Candidate {
21193                index: i,
21194                label: (b'A' + i as u8) as char,
21195                agent: "sonnet".to_owned(),
21196                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
21197                worktree: PathBuf::from(format!("/wt/{i}")),
21198                summary: String::new(),
21199                stat: String::new(),
21200                files: 0,
21201                commits: 0,
21202                empty,
21203                failed: None,
21204                verified_noop: verified.map(str::to_owned),
21205                duration_ms: 0,
21206                folded: false,
21207            })
21208            .collect();
21209    }
21210
21211    #[test]
21212    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
21213        ask_test_home();
21214        let mut runner = runner_at(RunStatus::Implementing);
21215        set_candidates(
21216            &mut runner,
21217            &[
21218                (true, Some("already on main at b32cfc4")),
21219                (true, Some("same fix, see the existing test")),
21220            ],
21221        );
21222
21223        runner
21224            .after_implement()
21225            .expect("a verified no-op is not an error");
21226
21227        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
21228    }
21229
21230    #[test]
21231    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
21232        ask_test_home();
21233        let mut runner = runner_at(RunStatus::Implementing);
21234        // Candidate A declares a verified no-op; candidate B simply wrote
21235        // nothing and said nothing about why. One candidate's claim is not
21236        // the whole run's agreement.
21237        set_candidates(
21238            &mut runner,
21239            &[(true, Some("already on main at b32cfc4")), (true, None)],
21240        );
21241
21242        let err = runner
21243            .after_implement()
21244            .expect_err("an unverified empty candidate must still fail the run");
21245
21246        assert!(
21247            err.to_string().contains("no candidate produced a change"),
21248            "{err}"
21249        );
21250        assert_eq!(runner.state.status, RunStatus::Failed);
21251    }
21252
21253    #[test]
21254    fn after_implement_still_fails_an_ordinary_all_empty_run() {
21255        ask_test_home();
21256        let mut runner = runner_at(RunStatus::Implementing);
21257        set_candidates(&mut runner, &[(true, None), (true, None)]);
21258
21259        let err = runner
21260            .after_implement()
21261            .expect_err("no candidate declared anything; this is an ordinary failure");
21262
21263        assert!(
21264            err.to_string().contains("no candidate produced a change"),
21265            "{err}"
21266        );
21267        assert_eq!(runner.state.status, RunStatus::Failed);
21268    }
21269}
21270
21271/// A graft rollback whose restore could not be confirmed (see
21272/// [`Runner::graft`]).
21273#[derive(Debug)]
21274struct GraftRestoreFailed(String);
21275
21276impl std::fmt::Display for GraftRestoreFailed {
21277    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
21278        write!(
21279            f,
21280            "could not restore the winner to its pre-graft commit: {}",
21281            self.0
21282        )
21283    }
21284}
21285
21286impl std::error::Error for GraftRestoreFailed {}
21287
21288/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
21289/// not the agent: they never draw on an agent's own timeout, and running out of
21290/// them never reads as an agent failure.
21291#[cfg(test)]
21292mod resource_wait_tests {
21293    use super::*;
21294    use std::collections::BTreeMap;
21295
21296    fn sh(id: &str, script: &str) -> AgentSpec {
21297        AgentSpec {
21298            id: id.to_owned(),
21299            kind: crate::config::AgentKind::Command,
21300            model: None,
21301            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
21302            extra_args: Vec::new(),
21303            env: BTreeMap::new(),
21304            prompt_delivery: None,
21305            max_turn_secs: None,
21306        }
21307    }
21308
21309    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
21310        SeatJob {
21311            seat: SeatState::new("impl-A", &spec.id, 1),
21312            spec,
21313            cwd: dir.to_path_buf(),
21314            prompt: "do it".to_owned(),
21315            timeout,
21316            allow_write: true,
21317            sessions: false,
21318            artifacts: dir.join("art"),
21319            stem: "impl-A".to_owned(),
21320            handover: None,
21321        }
21322    }
21323
21324    fn state(resource_wait_secs: u64) -> RunState {
21325        let mut config = Config::default();
21326        config.graph.timeout_resource_wait = resource_wait_secs;
21327        RunState::new(
21328            PathBuf::from("/repo"),
21329            "main".to_owned(),
21330            "abc1234".to_owned(),
21331            "task".to_owned(),
21332            config,
21333        )
21334    }
21335
21336    fn hold(cache: &Path) -> crate::cache::Guard {
21337        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
21338        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
21339            crate::cache::AcquireOutcome::Acquired(g) => g,
21340            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
21341        }
21342    }
21343
21344    async fn run(
21345        jobs: Vec<SeatJob>,
21346        sem: Arc<Semaphore>,
21347        cache: Option<&Path>,
21348        st: &mut RunState,
21349    ) -> Vec<(usize, SeatState, AgentOutcome)> {
21350        let prompts = Prompts::default();
21351        let ctx = WaveCtx {
21352            run: "r",
21353            node: "implement",
21354            prompts: &prompts,
21355            cache,
21356            round: None,
21357            carry_seats: false,
21358            pauses: None,
21359        };
21360        wave(jobs, sem, &ctx, st, 0).await
21361    }
21362
21363    #[tokio::test]
21364    async fn verification_cache_wait_preserves_the_command_budget() {
21365        crate::run::pin_test_home();
21366        let tmp = tempfile::tempdir().unwrap();
21367        let cache = tmp.path().join("target");
21368        let guard = hold(&cache);
21369        let release = tokio::spawn(async move {
21370            tokio::time::sleep(Duration::from_millis(100)).await;
21371            guard.release();
21372        });
21373        let mut st = state(10);
21374        let budget = Duration::from_millis(50);
21375        let wait = repository_wait(&st, None);
21376        let (outcomes, _) = with_cache_lease(
21377            &mut st,
21378            Some(&cache),
21379            "gate",
21380            "gate",
21381            tmp.path(),
21382            "abc1234",
21383            budget,
21384            "test gate",
21385            &wait,
21386            |_, available| async move {
21387                assert_eq!(
21388                    available, budget,
21389                    "contention must not shorten verification"
21390                );
21391                (Vec::new(), false, Vec::new())
21392            },
21393        )
21394        .await
21395        .unwrap();
21396        release.await.unwrap();
21397        assert!(outcomes.is_empty(), "{outcomes:?}");
21398        assert!(
21399            st.events
21400                .iter()
21401                .any(|event| event.message.contains("waiting for"))
21402        );
21403        assert!(st.resource_wait.is_none());
21404        // The verification body no longer owns the lease after returning.
21405        hold(&cache).release();
21406    }
21407
21408    #[tokio::test]
21409    async fn verification_cache_wait_expires_without_starting_commands() {
21410        crate::run::pin_test_home();
21411        let tmp = tempfile::tempdir().unwrap();
21412        let cache = tmp.path().join("target");
21413        let _held = hold(&cache);
21414        let mut st = state(0);
21415        let wait = repository_wait(&st, None);
21416        let (outcomes, _) = with_cache_lease(
21417            &mut st,
21418            Some(&cache),
21419            "gate",
21420            "gate",
21421            tmp.path(),
21422            "abc1234",
21423            Duration::from_secs(60),
21424            "test gate",
21425            &wait,
21426            |_, _| async { panic!("commands must not start without the cache lease") },
21427        )
21428        .await
21429        .unwrap();
21430        assert_eq!(outcomes.len(), 1);
21431        assert!(outcomes[0].resource_blocked);
21432        assert!(st.resource_wait.is_none());
21433        assert!(matches!(
21434            st.resource_block.unwrap().cause,
21435            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
21436        ));
21437    }
21438
21439    #[tokio::test]
21440    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
21441        crate::run::pin_test_home();
21442        let tmp = tempfile::tempdir().unwrap();
21443        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
21444        let cache = tmp.path().join("cache");
21445        let mut state = state(21600);
21446        let wait = repository_wait(&state, None);
21447        let (outcomes, _) = with_cache_lease(
21448            &mut state,
21449            Some(&cache),
21450            "gate",
21451            "gate",
21452            tmp.path(),
21453            "head",
21454            Duration::from_secs(60),
21455            "gate",
21456            &wait,
21457            |_, _| async { panic!("freshness failure must not start verification") },
21458        )
21459        .await
21460        .unwrap();
21461        assert_eq!(outcomes.len(), 1);
21462        assert!(outcomes[0].resource_blocked);
21463        let block = state.resource_block.unwrap();
21464        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
21465        assert!(!block.description().contains("timed out"));
21466    }
21467
21468    #[tokio::test]
21469    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
21470        crate::run::pin_test_home();
21471        let tmp = tempfile::tempdir().unwrap();
21472        let cache = tmp.path().join("cache");
21473        let _held = hold(&cache);
21474        let mut state = state(21600);
21475        let pause = Pause::new();
21476        let interrupt = Pause::new();
21477        let cause = ParkCause::Operator;
21478        pause.park_for(cause.clone());
21479        let prompts = Prompts::default();
21480        let ctx = WaveCtx {
21481            run: "test",
21482            node: "fix",
21483            prompts: &prompts,
21484            cache: Some(&cache),
21485            round: None,
21486            carry_seats: false,
21487            pauses: Some((&pause, &interrupt)),
21488        };
21489        let result = wave(
21490            vec![job(
21491                sh("writer", "exit 99"),
21492                tmp.path(),
21493                Duration::from_secs(1),
21494            )],
21495            Arc::new(Semaphore::new(1)),
21496            &ctx,
21497            &mut state,
21498            0,
21499        )
21500        .await;
21501        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
21502        assert_eq!(result[0].1.turns, 0);
21503        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21504        assert_eq!(
21505            state.resource_block.unwrap().cause,
21506            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
21507        );
21508        assert!(state.active.is_empty());
21509    }
21510
21511    #[tokio::test]
21512    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
21513        crate::run::pin_test_home();
21514        let tmp = tempfile::tempdir().unwrap();
21515        let cache = tmp.path().join("target-a");
21516        let guard = hold(&cache);
21517        // Held for longer than the agent's whole 1s budget.
21518        tokio::spawn(async move {
21519            tokio::time::sleep(Duration::from_millis(1500)).await;
21520            guard.release();
21521        });
21522        let mut st = state(60);
21523        let results = run(
21524            vec![job(
21525                sh("alpha", "echo ok"),
21526                tmp.path(),
21527                Duration::from_secs(1),
21528            )],
21529            Arc::new(Semaphore::new(1)),
21530            Some(&cache),
21531            &mut st,
21532        )
21533        .await;
21534        assert!(
21535            matches!(results[0].2, AgentOutcome::Ok(_)),
21536            "the wait must not be charged to the agent"
21537        );
21538        assert!(st.resource_block.is_none());
21539        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21540        assert!(st.resource_wait.is_none() && st.active.is_empty());
21541    }
21542
21543    #[tokio::test]
21544    #[cfg(unix)]
21545    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
21546        use std::os::fd::AsRawFd as _;
21547        crate::run::pin_test_home();
21548        let tmp = tempfile::tempdir().unwrap();
21549        crate::git::git(tmp.path(), &["init"]).await.unwrap();
21550        let common = tmp.path().join(".git");
21551        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
21552        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
21553        assert_eq!(
21554            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
21555            0
21556        );
21557        let mut st = state(1);
21558        st.config.coordination = crate::config::test_coordination();
21559        let mut reader = job(
21560            sh("reader", "echo must-not-run"),
21561            tmp.path(),
21562            Duration::from_secs(30),
21563        );
21564        reader.allow_write = false;
21565        let results = run(
21566            vec![reader.clone()],
21567            Arc::new(Semaphore::new(1)),
21568            None,
21569            &mut st,
21570        )
21571        .await;
21572        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21573        assert_eq!(results[0].1.turns, 0);
21574        assert!(FailClass::of(&results[0].2).is_none());
21575        assert_eq!(
21576            st.resource_block.as_ref().unwrap().kind,
21577            "repository coordination"
21578        );
21579        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21580        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21581        drop(external);
21582        let editor = crate::repo_lock::SeatLease::for_checkout(
21583            tmp.path(),
21584            true,
21585            crate::config::test_coordination().as_ref(),
21586        )
21587        .await
21588        .unwrap();
21589        let pause = Pause::new();
21590        let interrupt = Pause::new();
21591        let prompts = Prompts::default();
21592        let ctx = WaveCtx {
21593            run: "r",
21594            node: "review",
21595            prompts: &prompts,
21596            cache: None,
21597            round: None,
21598            carry_seats: false,
21599            pauses: Some((&pause, &interrupt)),
21600        };
21601        let (results, ()) = tokio::join!(
21602            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
21603            async {
21604                tokio::time::sleep(Duration::from_millis(50)).await;
21605                pause.park();
21606            }
21607        );
21608        assert!(
21609            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
21610        );
21611        assert_eq!(results[0].1.turns, 0);
21612        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21613        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21614        drop(editor);
21615    }
21616
21617    #[tokio::test]
21618    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
21619        crate::run::pin_test_home();
21620        let tmp = tempfile::tempdir().unwrap();
21621        let cache = tmp.path().join("target-b");
21622        let _held = hold(&cache);
21623        let mut st = state(1);
21624        let results = run(
21625            vec![job(
21626                sh("alpha", "echo ok"),
21627                tmp.path(),
21628                Duration::from_secs(1),
21629            )],
21630            Arc::new(Semaphore::new(1)),
21631            Some(&cache),
21632            &mut st,
21633        )
21634        .await;
21635        let (_, seat, out) = &results[0];
21636        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
21637        assert!(wave_blocked(&results));
21638        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
21639        assert_eq!(seat.turns, 0, "nothing was asked");
21640        assert!(
21641            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
21642            "no agent was spawned"
21643        );
21644        assert!(FailClass::of(out).is_none(), "not a failure class");
21645        let block = st.resource_block.as_ref().expect("recorded for the resume");
21646        assert_eq!(block.kind, "shared build cache");
21647        assert!(matches!(
21648            block.cause,
21649            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
21650        ));
21651        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21652        assert!(st.active.is_empty());
21653    }
21654
21655    #[tokio::test]
21656    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
21657        crate::run::pin_test_home();
21658        let tmp = tempfile::tempdir().unwrap();
21659        let mut st = state(1);
21660        let results = run(
21661            vec![job(
21662                sh("alpha", "echo ok"),
21663                tmp.path(),
21664                Duration::from_secs(1),
21665            )],
21666            Arc::new(Semaphore::new(0)),
21667            None,
21668            &mut st,
21669        )
21670        .await;
21671        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21672        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
21673        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
21674    }
21675
21676    #[tokio::test]
21677    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
21678        crate::run::pin_test_home();
21679        let tmp = tempfile::tempdir().unwrap();
21680        let mut st = state(60);
21681        let results = run(
21682            vec![job(
21683                sh("alpha", "sleep 30"),
21684                tmp.path(),
21685                Duration::from_secs(1),
21686            )],
21687            Arc::new(Semaphore::new(1)),
21688            None,
21689            &mut st,
21690        )
21691        .await;
21692        let out = &results[0].2;
21693        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
21694        let class = FailClass::of(out).expect("a failure");
21695        assert_eq!(class, FailClass::Timeout);
21696        assert!(should_hand_over(None, &class), "existing handover policy");
21697        assert!(st.resource_block.is_none());
21698    }
21699}