Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
534/// on an early return or a panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615impl Runner {
616    /// Start a fresh run against `repo`.
617    pub async fn start(
618        repo: &Path,
619        instruction: String,
620        config: Config,
621        origin: Origin,
622    ) -> Result<Self> {
623        Self::start_naming(repo, instruction, "", config, origin).await
624    }
625
626    /// [`Runner::start`] for a queued task. Titles and instructions never
627    /// select implementation sources. The title argument is retained for callers.
628    pub async fn start_naming(
629        repo: &Path,
630        instruction: String,
631        _also_scan: &str,
632        config: Config,
633        origin: Origin,
634    ) -> Result<Self> {
635        let repo = git::toplevel(repo).await?;
636        let missing = agent::missing_programs(&config.agents);
637        if !missing.is_empty() {
638            bail!(
639                "these agent programs are not on PATH: {}. Fix the roster in \
640                 magi.toml or install them.",
641                missing.join(", ")
642            );
643        }
644        let base_branch =
645            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
646                .await?;
647        let fetched_base = resolve_base(
648            &repo,
649            &base_branch,
650            &config.merge.remote,
651            config.coordination.as_ref(),
652        )
653        .await?;
654        let base_commit = fetched_base.commit.clone();
655        let roles = config.resolve_roles()?;
656        let max_parallel = config.graph.max_parallel.max(1);
657        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
658        // Recorded before the first save, so a crash right after minting
659        // cannot leave a run with no origin. Legibility only: nothing reads it
660        // to decide anything.
661        state.fetched_base = Some(fetched_base);
662        state.origin = Some(origin);
663        state.reserve()?;
664        state.event("start", format!("run {} created", state.id));
665        state.save()?;
666        Ok(Self {
667            state,
668            roles,
669            sem: Arc::new(Semaphore::new(max_parallel)),
670            pause: Pause::new(),
671            interrupt: Pause::new(),
672        })
673    }
674
675    /// Open a review-only run against work that already exists on `branch`.
676    ///
677    /// The expensive half of the graph is the implement wave — measured at
678    /// 111 and 134 internal tool-loop turns on this repository, against a
679    /// handful for a judge or a reviewer. The cheap half is worth running on
680    /// hand-written work too, and there was no way to reach it.
681    ///
682    /// No new state and no schema change are needed: a run with **one** viable
683    /// candidate and a tally already decided degrades `execute` to exactly
684    /// review → gate → merge, because `judge` skips a single-candidate field,
685    /// `deliberate` has fewer than two first choices to reconcile, `vote`
686    /// returns early, `tally` is already present and `fold_losers` has no
687    /// losers. Resuming such a run therefore does the right thing as well.
688    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
689        Self::review_taking_over(repo, branch, config, None, origin).await
690    }
691
692    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
693    /// at the same task still has `branch` checked out, its worktree is
694    /// released first if that is safe (see [`crate::handover`]), and the
695    /// review refuses with the reason if it is not. `None` is a hand-run
696    /// review: it has no earlier attempts, so only a worktree of a dead run
697    /// magi recorded itself can be released.
698    pub async fn review_taking_over(
699        repo: &Path,
700        branch: &str,
701        config: Config,
702        takeover: Option<crate::handover::Takeover>,
703        origin: Origin,
704    ) -> Result<Self> {
705        let repo = git::toplevel(repo).await?;
706        let missing = agent::missing_programs(&config.agents);
707        if !missing.is_empty() {
708            bail!(
709                "these agent programs are not on PATH: {}. Fix the roster in \
710                 magi.toml or install them.",
711                missing.join(", ")
712            );
713        }
714        let base_branch =
715            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
716                .await?;
717        if base_branch == branch {
718            bail!("`{branch}` is the base branch; there is nothing to review against");
719        }
720        let fetched_base = resolve_base(
721            &repo,
722            &base_branch,
723            &config.merge.remote,
724            config.coordination.as_ref(),
725        )
726        .await?;
727        let base_commit = fetched_base.commit.clone();
728
729        let roles = config.resolve_roles()?;
730        let max_parallel = config.graph.max_parallel.max(1);
731        let mut state = RunState::new(
732            repo.clone(),
733            base_branch,
734            base_commit.clone(),
735            String::new(),
736            config,
737        );
738        state.fetched_base = Some(fetched_base);
739        state.origin = Some(origin);
740        state.reserve()?;
741
742        // Released before anything else touches the branch: a stale local
743        // branch is moved with `git branch -f`, which git refuses while an
744        // earlier attempt's worktree still has it checked out. Everything
745        // after this point that can fail puts the old run back.
746        // A hand-run review has no task, hence no earlier attempts, but a
747        // worktree of a dead run magi made may still be released.
748        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
749            earlier: Vec::new(),
750            home: crate::run::home(),
751            choice: None,
752            task_instruction: None,
753        });
754        let wait = repository_wait(&state, None);
755        let coordination_config = state.config.coordination.clone();
756        let coordination =
757            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
758                .await?;
759        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
760        if let Some(released) = &released {
761            state.event(
762                "release",
763                format!(
764                    "took `{branch}` over from run {}: its worktree was released: {}",
765                    crate::run::short_of(&released.old_id),
766                    released.audit
767                ),
768            );
769        }
770        // The owner's answer to an earlier divergence question is applied
771        // here: after the release (git will not move a checked-out branch)
772        // and before the sync that would otherwise ask again.
773        state.foreign_checkout = crate::handover::preflight_review(
774            &repo,
775            branch,
776            &takeover.home,
777            &state.config.merge.remote,
778        )
779        .await?;
780        if state.foreign_checkout.is_some() && takeover.choice.is_some() {
781            bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
782        }
783        if let Some(choice) = takeover.choice.as_ref()
784            && let Err(e) =
785                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
786                    .await
787        {
788            if let Some(released) = &released {
789                released.restore(&repo, branch).await;
790            }
791            return Err(e.context("applying the owner's answer about the diverged branch"));
792        }
793        // An earlier attempt of this task that published the task head, and
794        // whose work is the branch under review, hands that head over: the
795        // review run updates the same pull request. Without one (a legacy
796        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
797        // own head and its own open pull request is adopted as before.
798        for id in takeover.earlier.iter().rev() {
799            if let Ok(old) = RunState::load_under(id, &takeover.home)
800                && let Some(head) = old.pr_head.clone()
801                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
802            {
803                state.pr_head = Some(head);
804                break;
805            }
806        }
807        drop(coordination);
808        let opened = Self::open_review(
809            &repo,
810            branch,
811            state,
812            roles,
813            max_parallel,
814            base_commit,
815            &takeover,
816        )
817        .await;
818        if opened.is_err()
819            && let Some(released) = &released
820        {
821            let _coordination = crate::repo_lock::SeatLease::operation_with(
822                &repo,
823                coordination_config.as_ref(),
824                &wait,
825            )
826            .await?;
827            released.restore(&repo, branch).await;
828        }
829        opened
830    }
831
832    /// Recover exactly the operator-selected public implementation, without touching old sources.
833    pub async fn recover_pr(
834        repo: &Path,
835        task: &str,
836        selection: crate::pr_recovery::Selection,
837        mut config: Config,
838    ) -> Result<Self> {
839        selection.configure(&mut config)?;
840        selection.check_task(task, &selection.instruction)?;
841        let base_branch =
842            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
843                .await?;
844        let fetched_base = resolve_base(
845            repo,
846            &base_branch,
847            &config.merge.remote,
848            config.coordination.as_ref(),
849        )
850        .await?;
851        let base_commit = fetched_base.commit.clone();
852        selection
853            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
854            .await?;
855        Box::pin(selection.verify_source(repo)).await?;
856        if let Some(proof) = &selection.source_proof {
857            anyhow::ensure!(
858                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
859                "source rebase base is outside the fetched landing base"
860            );
861        }
862        let roles = config.resolve_roles()?;
863        let max_parallel = config.graph.max_parallel.max(1);
864        let mut state = RunState::new(
865            repo.to_path_buf(),
866            base_branch,
867            base_commit.clone(),
868            selection.instruction.clone(),
869            config,
870        );
871        state.fetched_base = Some(fetched_base);
872        state.origin = Some(crate::run::Origin::queue(task));
873        state.pr_head = Some(selection.branch.clone());
874        state.pr_recovery = Some(selection.clone());
875        state.reserve()?;
876        if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
877            // Never leave a reserved record behind as a phantom owner of the head.
878            state.discard_reserved();
879            return Err(e);
880        }
881        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
882    }
883
884    async fn open_selected(
885        repo: &Path,
886        state: RunState,
887        roles: ResolvedRoles,
888        max_parallel: usize,
889        base_commit: String,
890        selection: crate::pr_recovery::Selection,
891    ) -> Result<Self> {
892        let mut state = state;
893        let branch = state.branch_for('A');
894        let wait = repository_wait(&state, None);
895        let prepared = async {
896            let coordination = coordinate_operation(&mut state, repo, &wait).await?;
897            git::git(repo, &["branch", &branch, selection.source()]).await?;
898            drop(coordination);
899            anyhow::Ok(())
900        }
901        .await;
902        if let Err(e) = prepared {
903            // Nothing was exposed yet: do not leave a reserved record as a
904            // phantom owner of the public head.
905            state.discard_reserved();
906            return Err(e);
907        }
908        state.event(
909            "recovery",
910            format!(
911                "selected implementation {} for {} (public lease {})",
912                selection.source(),
913                selection.url,
914                selection.head
915            ),
916        );
917        let takeover = crate::handover::Takeover {
918            earlier: Vec::new(),
919            home: crate::run::home(),
920            choice: None,
921            task_instruction: Some(selection.instruction.clone()),
922        };
923        if let Err(e) = state.save() {
924            state.discard_reserved();
925            return Err(e);
926        }
927        let id = state.id.clone();
928        let opened = Self::open_review(
929            repo,
930            &branch,
931            state,
932            roles,
933            max_parallel,
934            base_commit,
935            &takeover,
936        )
937        .await;
938        match opened {
939            Ok(runner) => Ok(runner),
940            Err(e) => {
941                if let Ok(mut partial) = RunState::load(&id) {
942                    partial.status = RunStatus::Blocked;
943                    partial.driver_exited = true;
944                    partial.event(
945                        "recovery",
946                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
947                    );
948                    partial.save()?;
949                }
950                Err(crate::pr_recovery::StartupFailed {
951                    run: id,
952                    cause: format!("{e:#}"),
953                }
954                .into())
955            }
956        }
957    }
958
959    /// What the newest earlier attempt that reviewed `branch` leaves for this
960    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
961    /// no earlier attempts. Reads the earlier record only: ownership of its
962    /// worktree and branch is [`crate::handover`]'s business.
963    async fn inherit(
964        repo: &Path,
965        branch: &str,
966        base_commit: &str,
967        takeover: &crate::handover::Takeover,
968    ) -> Option<crate::recovery::RecoveryRecord> {
969        use crate::recovery::{Relation, assess};
970        let newest = takeover.earlier.last()?;
971        let mut chosen = None;
972        for id in takeover.earlier.iter().rev() {
973            if let Ok(old) = RunState::load_under(id, &takeover.home)
974                && !old.reviews.is_empty()
975                && (old.winner().is_some_and(|w| w.branch == branch)
976                    || old.pr_head.as_deref() == Some(branch))
977            {
978                chosen = Some(old);
979                break;
980            }
981        }
982        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
983            .await
984            .unwrap_or_default();
985        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
986            Some(last) if last.head == tip => Relation::Same,
987            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
988            _ => Relation::Diverged,
989        };
990        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
991        Some(assess(from, chosen.as_ref(), base_commit, relation))
992    }
993
994    /// The half of [`Runner::review_taking_over`] that can fail after an
995    /// earlier attempt's worktree was released.
996    async fn open_review(
997        repo: &Path,
998        branch: &str,
999        mut state: RunState,
1000        roles: ResolvedRoles,
1001        max_parallel: usize,
1002        base_commit: String,
1003        takeover: &crate::handover::Takeover,
1004    ) -> Result<Self> {
1005        let wait = repository_wait(&state, None);
1006        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1007        if state.foreign_checkout.is_some() {
1008            // Never reconcile or force-move a ref checked out by the operator.
1009            if state.pr_head.is_none() {
1010                state.pr_head = Some(branch.to_owned());
1011            }
1012        } else if state.pr_recovery.is_none() {
1013            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1014        }
1015        // The commit subjects are the closest thing to a task statement that
1016        // existing work carries, and the reviewers are told as much.
1017        let start = review_base(
1018            repo,
1019            &state.config.merge.remote,
1020            &state.base_branch,
1021            &base_commit,
1022            branch,
1023        )
1024        .await;
1025        let log = git::log_oneline(repo, &start, branch)
1026            .await
1027            .unwrap_or_default();
1028        let log_text = if log.trim().is_empty() {
1029            "(no commit messages)"
1030        } else {
1031            log.trim()
1032        };
1033        let mut instruction = format!(
1034            "Review the work already on branch `{branch}`. There is no task \
1035             statement: what the change claims to do is whatever its commits \
1036             say.\n\n{log_text}"
1037        );
1038        // A retry of a task whose earlier attempt left reviews behind carries
1039        // that attempt's evidence, and the task's own statement with it: the
1040        // completion criteria are the task's, not whatever the commits claim.
1041        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1042            if let Some(task) = takeover
1043                .task_instruction
1044                .as_deref()
1045                .filter(|t| !t.trim().is_empty())
1046                && rec.reviewed_head.is_some()
1047            {
1048                instruction = format!(
1049                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1050                    task.trim()
1051                );
1052                rec.criteria_inherited = true;
1053            }
1054            state.event("recovery", rec.summary());
1055            state.recovery = Some(rec);
1056        }
1057        if let Some(selected) = &state.pr_recovery {
1058            instruction = selected.instruction.clone();
1059        }
1060        state.task_instruction = takeover.task_instruction.clone();
1061        state.instruction = instruction;
1062        state.reviewed_commits = Some(
1063            git::subjects(repo, &start, branch)
1064                .await
1065                .unwrap_or_default(),
1066        );
1067
1068        // A takeover must not forget a hold that a previous review observed,
1069        // even if repairs have since removed it from the branch.
1070        for id in &takeover.earlier {
1071            if let Ok(old) = RunState::load_under(id, &takeover.home)
1072                && old.repo == state.repo
1073                && (old.pr_head.as_deref() == Some(branch)
1074                    || old.winner().is_some_and(|w| w.branch == branch))
1075            {
1076                for hold in old.branch_owner_holds {
1077                    if !state.branch_owner_holds.contains(&hold) {
1078                        state.event(
1079                            "approval-policy",
1080                            format!(
1081                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1082                                hold.control.key(),
1083                                hold.file,
1084                                hold.head
1085                            ),
1086                        );
1087                        state.branch_owner_holds.push(hold);
1088                    }
1089                }
1090            }
1091        }
1092        let reviewed_head = git::rev_parse(repo, branch).await?;
1093        state.observe_branch_owner_holds(&reviewed_head).await;
1094
1095        // An attached worktree, so the fixer's commits land on the branch under
1096        // review rather than on a detached head nobody will look at again.
1097        let worktree = state.worktree_root().join("under-review");
1098        if let Some(parent) = worktree.parent() {
1099            tokio::fs::create_dir_all(parent).await.ok();
1100        }
1101        let path = worktree.to_string_lossy().to_string();
1102        let review_branch = if let Some(checkout) = &state.foreign_checkout {
1103            let shadow = format!("magi/{}/review", state.short());
1104            let tracking = format!("{}/{branch}", state.config.merge.remote);
1105            // A surviving tracking ref is not evidence that the remote still
1106            // offers this tip. Adopt it only after a successful fetch here.
1107            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1108            let mut head = checkout.synced_head.clone();
1109            if matches!(fetched, Ok(out) if out.ok()) {
1110                let remote = git::rev_parse(repo, &tracking).await?;
1111                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1112                    head = remote;
1113                }
1114            }
1115            git::git(repo, &["branch", &shadow, &head]).await?;
1116            shadow
1117        } else {
1118            branch.to_owned()
1119        };
1120        git::git(repo, &["worktree", "add", &path, &review_branch])
1121            .await
1122            .with_context(|| {
1123                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1124            })?;
1125
1126        let setup = match ensure_setup_config(&mut state, repo).await {
1127            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1128            Err(e) => Err(e),
1129        };
1130        if let Err(e) = setup {
1131            git::worktree_remove_clean(repo, &worktree).await.ok();
1132            return Err(e);
1133        }
1134        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1135            .await
1136            .unwrap_or(0);
1137        if commits == 0 {
1138            git::worktree_remove_clean(repo, &worktree).await.ok();
1139            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1140        }
1141        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1142            .await
1143            .map(|f| f.len())
1144            .unwrap_or(0);
1145        if files == 0
1146            && let (Ok(head_tree), Ok(base_tree)) = (
1147                git::tree_of(&worktree, "HEAD").await,
1148                git::tree_of(&worktree, &base_commit).await,
1149            )
1150            && head_tree == base_tree
1151        {
1152            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1153            git::worktree_remove_clean(repo, &worktree).await.ok();
1154            bail!(
1155                "`{branch}` at {} has a tree identical to base {}; this usually means \
1156                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1157                 against `{}/{branch}`) rather than an empty change",
1158                short(&head),
1159                short(&base_commit),
1160                state.config.merge.remote
1161            );
1162        }
1163        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1164            .await
1165            .unwrap_or_default();
1166
1167        state.candidates.push(Candidate {
1168            index: 0,
1169            label: 'A',
1170            // Not an agent id on purpose: nothing in the roster wrote this, and
1171            // the stats tables must not credit anyone with a win for it.
1172            agent: EXISTING_BRANCH.to_owned(),
1173            branch: review_branch,
1174            worktree,
1175            summary: String::new(),
1176            stat,
1177            files,
1178            commits,
1179            empty: false,
1180            failed: None,
1181            verified_noop: None,
1182            duration_ms: 0,
1183            folded: false,
1184        });
1185        state.tally = Some(Tally {
1186            first_choice: BTreeMap::from([('A', 0)]),
1187            borda: BTreeMap::new(),
1188            winner: 'A',
1189            rankings: 0,
1190            unanimous_initial: false,
1191            deliberated: false,
1192            changed_votes: 0,
1193            unanimous_final: false,
1194            tie_break: None,
1195            // No panel sat, so no quorum applies. Zero judges is the correct
1196            // number for work that never competed, and must not be reported as
1197            // a collapsed panel.
1198            judges: 0,
1199            present: 0,
1200            quorum: 0,
1201            met_quorum: true,
1202            uncontested: Some("review-only run: nothing competed".to_owned()),
1203        });
1204        state.status = RunStatus::Reviewing;
1205        state.prep_complete = Some(true);
1206        state.event(
1207            "start",
1208            format!(
1209                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1210                state.id
1211            ),
1212        );
1213        state.save()?;
1214        Ok(Self {
1215            state,
1216            roles,
1217            sem: Arc::new(Semaphore::new(max_parallel)),
1218            pause: Pause::new(),
1219            interrupt: Pause::new(),
1220        })
1221    }
1222
1223    /// Authorize exactly one additional rebase, under the caller's queue task
1224    /// claim and the same owner/execution locks used by run maintenance.
1225    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1226        Self::authorize_base_recovery_request(home, id, task, false).await
1227    }
1228
1229    /// Queue release uses its durable approval generation exactly once.
1230    pub(crate) async fn authorize_base_recovery_from_resumption(
1231        home: &Path,
1232        id: &str,
1233        task: &str,
1234    ) -> Result<()> {
1235        Self::authorize_base_recovery_request(home, id, task, true).await
1236    }
1237
1238    async fn authorize_base_recovery_request(
1239        home: &Path,
1240        id: &str,
1241        task: &str,
1242        resumption: bool,
1243    ) -> Result<()> {
1244        let _owner = crate::pr_head::owner_lock(home, id)?;
1245        let mut state = RunState::load_under(id, home)?;
1246        crate::pr_head::ensure_owner(&state)?;
1247        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1248        anyhow::ensure!(
1249            queued.runs.last().is_some_and(|run| run == id)
1250                && queued.status != crate::queue::TaskStatus::Done
1251                && !queued.fresh_start
1252                && queued.review_branch.is_none(),
1253            "run task ownership changed"
1254        );
1255        anyhow::ensure!(
1256            state
1257                .origin
1258                .as_ref()
1259                .and_then(|o| o.task.as_deref())
1260                .is_none_or(|owner| owner == task),
1261            "run belongs to another task"
1262        );
1263        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1264        anyhow::ensure!(
1265            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1266            "run is live or its ownership is unknown"
1267        );
1268        // Judge the previous driver's lease before acquiring our maintenance
1269        // lease. The owner lock excludes new driver claims across both steps.
1270        let _execution = crate::driver::maintenance(home, id)?;
1271        anyhow::ensure!(
1272            state.status.resumable()
1273                && state.released_to.is_none()
1274                && state.retired_to.is_none()
1275                && state.pr_taken_over_by.is_none(),
1276            "run ownership is final"
1277        );
1278        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1279        anyhow::ensure!(
1280            !state.pr_publication_pending
1281                && !crate::handover::landing_checkpoint(&state)
1282                && state.merge_decision.is_none(),
1283            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1284        );
1285        let winner = state.winner().context("run has no retained winner")?;
1286        anyhow::ensure!(
1287            git::is_clean(&winner.worktree).await?,
1288            "retained worktree has uncommitted changes"
1289        );
1290        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1291        anyhow::ensure!(
1292            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1293            "retained worktree does not match the winner branch"
1294        );
1295        state.record_base_sync_stop(&head);
1296        state.save()?;
1297        if resumption {
1298            anyhow::ensure!(
1299                crate::daemon::compatible_run_instruction(&state, &queued),
1300                "resumption instruction does not match the retained run"
1301            );
1302            let approval = queued
1303                .resumption
1304                .as_ref()
1305                .context("no operator resumption")?;
1306            anyhow::ensure!(
1307                approval.instruction == queued.instruction,
1308                "resumption approval does not match the current task instruction"
1309            );
1310            state.authorize_base_sync_resumption(task, approval)?;
1311        } else {
1312            state.authorize_base_sync(task)?;
1313        }
1314        state.event(
1315            "base-recovery",
1316            "owner authorized one additional base rebase; fresh verification required",
1317        );
1318        state.save()?;
1319        Ok(())
1320    }
1321
1322    /// Reopen an existing run.
1323    pub fn resume(id: &str) -> Result<Self> {
1324        let state = RunState::load(id)?;
1325        crate::pr_head::ensure_owner(&state)?;
1326        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1327            bail!(
1328                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1329            );
1330        }
1331        if let Some(to) = &state.released_to {
1332            bail!(
1333                "run {} cannot be resumed: its worktree was released to run {}",
1334                state.short(),
1335                crate::run::short_of(to)
1336            );
1337        }
1338        let roles = state.config.resolve_roles()?;
1339        if let Some(id) = &state.config.graph.selected_implementer
1340            && state.candidates.iter().any(|c| c.agent != *id)
1341        {
1342            bail!(
1343                "replacement implementer pin does not match the saved candidates; task must stay held"
1344            );
1345        }
1346        let max_parallel = state.config.graph.max_parallel.max(1);
1347        Ok(Self {
1348            state,
1349            roles,
1350            sem: Arc::new(Semaphore::new(max_parallel)),
1351            pause: Pause::new(),
1352            interrupt: Pause::new(),
1353        })
1354    }
1355
1356    /// Walk the graph to a terminal state, skipping nodes already recorded.
1357    ///
1358    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1359    /// the phone - ends here, so this is the one place a run that ended
1360    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1361    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1362    pub async fn execute(&mut self) -> Result<()> {
1363        if let Some(selection) = &self.state.pr_recovery
1364            && (selection.source_proof.is_some() || selection.source_run.is_some())
1365        {
1366            Box::pin(selection.verify_source(&self.state.repo)).await?;
1367        }
1368        // Queue claims cover daemon work; this also serializes manual drivers
1369        // consuming a repair answer, including the new review and decision.
1370        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1371            || self.state.land_approval.is_some()
1372            || self.state.merge_decision.is_some()
1373            || !self.state.merge_repairs.is_empty()
1374            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1375        {
1376            let claim = FixClaim::acquire(&self.state.dir())?;
1377            let disk = RunState::load(&self.state.id)?;
1378            let checkpoint = |s: &RunState| {
1379                (
1380                    s.merge_repairs.len(),
1381                    s.merge_repairs
1382                        .last()
1383                        .map(|r| (r.question.clone(), r.phase)),
1384                    s.land_approval
1385                        .as_ref()
1386                        .map(|a| (a.question.clone(), a.head.clone())),
1387                )
1388            };
1389            if checkpoint(&disk) != checkpoint(&self.state) {
1390                bail!(
1391                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1392                );
1393            }
1394            Some(claim)
1395        } else {
1396            None
1397        };
1398        let mut execution = self.claim_driver().await?;
1399        let runner = &mut *execution.runner;
1400        let wait = runner.repository_wait();
1401        // Keep the graph's large future off the callers' stack, including
1402        // standalone task attempts running on a default-sized test thread.
1403        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1404        let result = match result {
1405            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1406                record_repository_block(&mut runner.state, &e);
1407                runner.park_here()?;
1408                Ok(())
1409            }
1410            other => other,
1411        };
1412        execution.driver.finish(&mut runner.state).await?;
1413        let ended = if result.is_err() {
1414            Some({
1415                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1416                if let Err(error) = &result {
1417                    notice.message = format!(
1418                        "Run {} stopped with an error: {}",
1419                        runner.state.id,
1420                        error.to_string().lines().next().unwrap_or("unknown error")
1421                    );
1422                }
1423                notice
1424            })
1425        } else {
1426            crate::notices::run_ended(&runner.state)
1427        };
1428        if let Some(notice) = ended {
1429            crate::notices::raise(notice);
1430        }
1431        result
1432    }
1433
1434    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1435        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1436    }
1437
1438    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1439        let wait = self.repository_wait();
1440        coordinate_operation(&mut self.state, cwd, &wait).await
1441    }
1442
1443    /// Validate ownership and persist the driver claim in one critical section.
1444    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1445        let owner = match crate::pr_head::owner_lock_wait(
1446            &crate::run::home(),
1447            &self.state.id,
1448            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1449        )
1450        .await
1451        {
1452            Ok(owner) => owner,
1453            Err(error) => {
1454                let detail = format!("worktree maintenance or ownership update: {error:#}");
1455                // Admission did not own run.json: retain this resource outcome
1456                // in memory for queue settlement without overwriting a concurrent
1457                // retirement or publication transaction.
1458                self.state.resource_block = Some(crate::run::ResourceBlock {
1459                    cause: crate::run::ResourceBlockCause::Other,
1460                    node: "start".into(),
1461                    kind: "worktree maintenance".into(),
1462                    detail: detail.clone(),
1463                    at: Timestamp::now(),
1464                });
1465                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1466            }
1467        };
1468        let disk = RunState::load(&self.state.id)?;
1469        crate::pr_head::ensure_owner(&disk)?;
1470        let claimed =
1471            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1472        // Only proven-live ownership refuses here. Unknown records (legacy runs
1473        // with no claim marker or process identity) fall through to the
1474        // exclusive execution lease taken just below, which is what actually
1475        // keeps a second driver out.
1476        anyhow::ensure!(
1477            disk.liveness(claimed) != crate::run::Liveness::Live,
1478            "run already has a live driver; wait for its execution to end"
1479        );
1480        if let Some(to) = &disk.released_to {
1481            bail!(
1482                "run {} cannot continue: its worktree was released to run {}",
1483                self.state.short(),
1484                crate::run::short_of(to)
1485            );
1486        }
1487        anyhow::ensure!(
1488            serde_json::to_value(&disk.base_sync_recovery)?
1489                == serde_json::to_value(&self.state.base_sync_recovery)?,
1490            "base recovery changed since this run was loaded; reload the saved run"
1491        );
1492        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1493        let mut execution = DriverExecution {
1494            runner: self,
1495            driver,
1496        };
1497        // Declare after the execution guard: on any claim error this short
1498        // lock must drop before that guard reloads and releases ownership.
1499        let _owner = owner;
1500        let state = &mut execution.runner.state;
1501        // Moving again, so it is no longer parked. Set before the walk rather
1502        // than in `resume`, so every way of re-entering the graph clears it
1503        // and a card cannot claim a run is waiting to be resumed while the
1504        // agents are already working.
1505        state.parked = false;
1506        state.resource_wait = None;
1507        state.resource_block = None;
1508        // Any seat this state still lists as answering belongs to whatever
1509        // process last drove this run — this one included, if it crashed
1510        // mid-wave. Cleared and flushed immediately, before anything else
1511        // runs, so a resume can never show a seat as live when nothing is
1512        // asking it anything yet; the node that actually dispatches the next
1513        // wave repopulates it.
1514        state.clear_active();
1515        // Recorded in the same spot, and flushed together with the clear
1516        // above: this is the pid a reader checks (`RunState::liveness`) when
1517        // no daemon claim exists to answer "is a process still driving this
1518        // run" — a plain `magi run` / `magi review` typed into a terminal
1519        // claims nothing there. Always overwritten, never only-if-absent, so
1520        // a resumed run's stale pid from a previous, possibly-dead process
1521        // can never survive into this one's own report. Unlike
1522        // `clear_active`, this changes on every single `execute()` call, so
1523        // the save below is now unconditional rather than only-if-cleared.
1524        //
1525        // `driver_started_at` is recorded in the same breath, from this same
1526        // pid, so `liveness` can tell a live pid that is genuinely still us
1527        // apart from one the OS has since handed to an unrelated process —
1528        // see that field's own doc for why the pid alone is not enough.
1529        let pid = std::process::id();
1530        state.driver_pid = Some(pid);
1531        state.driver_started_at = crate::proc::process_started_at(pid);
1532        state.driver_exited = false;
1533        state.driver_claimed = Some(true);
1534        state.begin_pass();
1535        execution.driver.bind(state);
1536        state.save()?;
1537        execution.driver.record(state)?;
1538        Ok(execution)
1539    }
1540
1541    async fn execute_graph(&mut self) -> Result<()> {
1542        self.state.configure_recovery_resume()?;
1543        // Preservation may invalidate an old empty classification. Resolve it
1544        // before any resume shortcut can reuse a verdict or publish a branch.
1545        self.persist_candidates().await?;
1546        // A saved base stop outranks stage markers, including legacy Voting.
1547        // Recovery never reenters implementation, deliberation or voting.
1548        if self.state.base_sync_blocked() {
1549            self.sync_to_base().await?;
1550            if !self.state.base_sync_blocked()
1551                && self.state.status != RunStatus::AlreadyInBase
1552                && !self.resource_halt()?
1553            {
1554                self.finish_after_tally().await?;
1555            }
1556            return Ok(());
1557        }
1558        if self
1559            .state
1560            .merge_decision
1561            .as_ref()
1562            .is_some_and(|decision| !decision.closed)
1563            && self.state.status != RunStatus::Landing
1564        {
1565            if crate::merge_decision::resumable_hold(&self.state) {
1566                self.state.merge = Some(MergeOutcome {
1567                    mode: MergeMode::Pr,
1568                    ok: false,
1569                    detail: self
1570                        .state
1571                        .merge_decision
1572                        .as_ref()
1573                        .unwrap()
1574                        .target
1575                        .pr
1576                        .clone(),
1577                    empty: false,
1578                });
1579                self.run_land().await?;
1580                self.settle_questions();
1581                return Ok(());
1582            }
1583            bail!(
1584                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1585            );
1586        }
1587        if self.state.config.graph.selected_implementer.is_some()
1588            && self.state.status == RunStatus::Blocked
1589            && self.state.tally.is_none()
1590            && self.state.candidate_preservation.is_empty()
1591            && self.state.interrupted_implementations.is_empty()
1592        {
1593            bail!(
1594                "selected implementation is held; explicitly release the owning task to retry it"
1595            );
1596        }
1597
1598        if self.state.merge_repairs.last().is_some_and(|r| {
1599            matches!(
1600                r.phase,
1601                crate::run::MergeRepairPhase::Requested
1602                    | crate::run::MergeRepairPhase::Fixing
1603                    | crate::run::MergeRepairPhase::Reviewing
1604            )
1605        }) {
1606            if self.repair_merge_review().await? {
1607                self.run_land().await?;
1608            }
1609            self.settle_questions();
1610            return Ok(());
1611        }
1612        // Publication already passed review and gate. Resume only that node:
1613        // syncing to a newer base here could change the verified commit.
1614        if self.state.pr_publication_pending {
1615            self.merge().await?;
1616            if self.resource_halt()? {
1617                return Ok(());
1618            }
1619            self.state.save()?;
1620            self.settle_questions();
1621            return Ok(());
1622        }
1623        // A run that already lost its quorum never resumes into the verdict
1624        // machinery: `deliberate` and `vote` would otherwise clobber the
1625        // stalled marker back to Voting and the run would keep going past a
1626        // verdict that is no longer trustworthy. Everything already recorded is
1627        // kept, so the run stays resumable (or foldable) for a human to pick up.
1628        //
1629        // On --resume the run gets one chance to repair itself: the seats a
1630        // rate limit took out are re-asked. If their quota has since reset and
1631        // the quorum is restored, the run picks up and finishes; otherwise it
1632        // stays stale and still-resumable for a later retry. If it does not
1633        // recover, the returned status stays `Stalled` and nothing was
1634        // clobbered (the recovery only mutates entries for the lost seats).
1635        if self.state.status == RunStatus::Stalled {
1636            if self.recover_stall().await? {
1637                self.finish_after_tally().await?;
1638            } else {
1639                // Still below quorum: persist the marker and stay resumable.
1640                self.state.save()?;
1641            }
1642            return Ok(());
1643        }
1644        // A run parked inside `land` - watching CI, mid fix-round, or
1645        // waiting on the owner's merge approval - resumes directly into it,
1646        // never back through `prep`. Everything before `merge` already
1647        // concluded; that is the only way `status` reaches `Landing` in the
1648        // first place. Re-walking `review_loop` first would also be actively
1649        // wrong: its own status recomputation (see its doc) treats any
1650        // clean round as reason to set `status` to `Gating`, which would
1651        // clobber this marker before `merge` ever ran, and this run would
1652        // never find its way back into `land` at all.
1653        if self.state.status == RunStatus::Landing
1654            || crate::handover::landing_checkpoint(&self.state)
1655        {
1656            self.state.land_sync_pending = true;
1657            self.state.save()?;
1658            self.run_land().await?;
1659            // `run_land` may have settled the run right here - CI came back
1660            // green and the PR merged, say - without ever passing back
1661            // through `merge`'s own trailing call. Whatever it left `status`
1662            // as is what this has to read.
1663            self.settle_questions();
1664            return Ok(());
1665        }
1666        // A run parked at `merge` on the owner's word about a withheld PR text
1667        // resumes straight into `merge`. Re-walking the tail would sync to the
1668        // base again (possibly rebasing, possibly spending the fixer) after the
1669        // gate had passed, and then push a tree nobody reviewed or gated.
1670        if self.state.github_text.is_some()
1671            && self.state.merge.is_none()
1672            && self.state.status == RunStatus::Gating
1673        {
1674            self.merge().await?;
1675            if self.resource_halt()? {
1676                return Ok(());
1677            }
1678            self.state.save()?;
1679            return Ok(());
1680        }
1681        if let Err(e) = self.prep().await {
1682            if e.is::<crate::repo_lock::ResourceBlocked>() {
1683                return Err(e);
1684            }
1685            self.state.status = RunStatus::Blocked;
1686            self.state
1687                .prep_failure
1688                .get_or_insert_with(|| format!("{e:#}"));
1689            self.state
1690                .event("prep", format!("preparation stopped: {e:#}"));
1691            self.state.save()?;
1692            return Err(e);
1693        }
1694        if self.park_here()? {
1695            return Ok(());
1696        }
1697        self.advise().await?;
1698        if self.park_here()? {
1699            return Ok(());
1700        }
1701        self.implement().await?;
1702        if self.park_here()? {
1703            return Ok(());
1704        }
1705        // `after_implement` already saved the state and settled any open
1706        // questions when it set this; nothing later in the graph has
1707        // anything to judge.
1708        if self.state.status == RunStatus::VerifiedNoop {
1709            return Ok(());
1710        }
1711        self.judge().await?;
1712        if self.park_here()? {
1713            return Ok(());
1714        }
1715        self.deliberate().await?;
1716        if self.park_here()? {
1717            return Ok(());
1718        }
1719        self.vote().await?;
1720        if self.park_here()? {
1721            return Ok(());
1722        }
1723        self.tally()?;
1724        // A verdict that lost its quorum is not trustworthy: do not review,
1725        // gate, or merge on it. Everything already done is kept, so the run
1726        // stays resumable (or foldable); the human can replace the agent that
1727        // ran out of quota and pick it up.
1728        if self.state.status == RunStatus::Stalled {
1729            // Persist the stalled marker now — the normal end-of-execute save
1730            // below is below this early return, and without it a resumed run
1731            // would reload a pre-tally status and keep going.
1732            self.state.save()?;
1733            return Ok(());
1734        }
1735        self.finish_after_tally().await?;
1736        Ok(())
1737    }
1738
1739    /// A wave could not start its agents because a shared resource stayed busy
1740    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1741    /// resumable. Unstarted seats have no result; completed peers retain
1742    /// their saved candidate state.
1743    fn stop_for_resource(&mut self) -> Result<()> {
1744        // Resource acquisition observes cancellation inside a node. Complete
1745        // the same durable park as every other node boundary before returning.
1746        if self.pause.parked() || self.interrupt.parked() {
1747            self.park_here()?;
1748            return Ok(());
1749        }
1750        if self.state.resource_block.is_some() {
1751            self.state.status = RunStatus::Blocked;
1752        }
1753        self.state.save()
1754    }
1755
1756    /// Stop walking the graph if a node ended on a resource block.
1757    fn resource_halt(&mut self) -> Result<bool> {
1758        if self.state.resource_block.is_none() {
1759            return Ok(false);
1760        }
1761        self.stop_for_resource()?;
1762        Ok(true)
1763    }
1764
1765    /// Park here if asked to, recording it in the run's own timeline.
1766    ///
1767    /// Returns whether the caller should stop walking the graph. The state is
1768    /// saved either way by the node that just finished; this adds the event so
1769    /// the operator's card says why a run that is neither finished nor moving
1770    /// is sitting where it is.
1771    fn park_here(&mut self) -> Result<bool> {
1772        let parking = self.pause.parked() || self.interrupt.parked();
1773        if !parking && self.state.resource_block.is_some() {
1774            self.stop_for_resource()?;
1775            return Ok(true);
1776        }
1777        // Either handle asking is enough - see `Pause`'s own doc for why
1778        // they are never the same one. `interrupt` is checked second so a
1779        // reason it carries is preferred in the message below over a plain
1780        // shutdown park racing it at the same boundary.
1781        if !self.pause.parked() && !self.interrupt.parked() {
1782            return Ok(false);
1783        }
1784        if self.state.parked {
1785            return Ok(true);
1786        }
1787        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1788        if let Some(block) = &mut self.state.resource_block
1789            && matches!(
1790                block.cause,
1791                crate::run::ResourceBlockCause::Cancelled { .. }
1792            )
1793        {
1794            block.cause = crate::run::ResourceBlockCause::Cancelled {
1795                park: cause.clone(),
1796            };
1797        }
1798        let why = match cause.as_ref().map(ParkCause::label) {
1799            Some(reason) => format!(
1800                "parked after `{}` ({reason}) — resume to carry on from here",
1801                self.state.status.as_str()
1802            ),
1803            None => format!(
1804                "parked after `{}` — resume to carry on from here",
1805                self.state.status.as_str()
1806            ),
1807        };
1808        self.state.event("park", why);
1809        self.state
1810            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1811        self.state.parked = true;
1812        self.state.save()?;
1813        Ok(true)
1814    }
1815
1816    /// Only a new explicit operator approval may retry failed preparation.
1817    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1818        if self.state.prep_failure.is_none() {
1819            return Ok(());
1820        }
1821        let failed_at = self
1822            .state
1823            .events
1824            .iter()
1825            .rev()
1826            .find(|event| event.node == "prep")
1827            .map_or(self.state.updated_at, |event| event.at);
1828        anyhow::ensure!(
1829            approval_at > failed_at,
1830            "preparation retry requires a new explicit release after the failure"
1831        );
1832        self.state.prep_failure = None;
1833        self.state.prep_complete = Some(false);
1834        self.state
1835            .event("prep-retry", "operator approved retry of preparation");
1836        Ok(())
1837    }
1838
1839    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1840    pub fn on_pause(&mut self, pause: Pause) {
1841        self.pause = pause;
1842    }
1843
1844    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1845    /// scheduler asking this one run - and no other - to park so a task
1846    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1847    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1848    /// handle.
1849    pub fn watch_interrupt(&mut self, pause: Pause) {
1850        self.interrupt = pause;
1851    }
1852
1853    /// Abandon this run's own open questions, once `status` has actually
1854    /// settled rather than merely paused.
1855    ///
1856    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1857    /// either back up with the candidates, the review round and the seat
1858    /// sessions already on disk, so a question an implementer asked mid-round
1859    /// may still get a real answer read by a real resume. Only the statuses
1860    /// `resumable` excludes are actually final: the run merged, it reached
1861    /// `Ready` with nothing left to do, it failed outright with no
1862    /// established point to continue from, or every candidate agreed, with
1863    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1864    /// every one of those the seat that asked is gone for good, exactly like
1865    /// the run being deleted under `magi run rm` - so the same cleanup
1866    /// applies, worded for what actually happened instead of "the run was
1867    /// deleted".
1868    ///
1869    /// Best-effort and silent on success: called from every place `status`
1870    /// can land on one of those three, including ones a resumed run revisits,
1871    /// so it must cost nothing when there was nothing open to begin with.
1872    fn settle_questions(&mut self) {
1873        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1874            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1875        }
1876    }
1877
1878    /// The tail of the graph after a trustworthy tally: fold losers, review,
1879    /// gate, merge, and persist.
1880    async fn finish_after_tally(&mut self) -> Result<()> {
1881        // Before the losers are folded: their branches are what the graft
1882        // reads from.
1883        self.graft().await?;
1884        if self.resource_halt()? {
1885            return Ok(());
1886        }
1887        self.fold_losers().await?;
1888        // Before review starts, and again right before the gate: a run's
1889        // review rounds can themselves take long enough for the base to move
1890        // a second time, and the gate is the one node whose "green" gets
1891        // acted on.
1892        self.sync_to_base().await?;
1893        if self.hold_base_sync().await? || self.resource_halt()? {
1894            return Ok(());
1895        }
1896        if self.state.status == RunStatus::AlreadyInBase {
1897            return Ok(());
1898        }
1899        self.review_loop().await?;
1900        if self.resource_halt()? {
1901            return Ok(());
1902        }
1903        self.sync_to_base().await?;
1904        if self.hold_base_sync().await? || self.resource_halt()? {
1905            return Ok(());
1906        }
1907        if self.state.status == RunStatus::AlreadyInBase {
1908            return Ok(());
1909        }
1910        self.gate().await?;
1911        self.merge().await?;
1912        if self.resource_halt()? {
1913            return Ok(());
1914        }
1915        self.state.save()?;
1916        Ok(())
1917    }
1918
1919    // ---------------------------------------------------------------- prep
1920
1921    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1922    /// run with the step and its output in the event; it is never swallowed.
1923    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1924    /// resume starts over instead of finding candidates and missing seats.
1925    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1926        let result = match ensure_setup_config(&mut self.state, repo).await {
1927            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1928            Err(e) => Err(e),
1929        };
1930        if let Err(e) = result {
1931            git::worktree_remove_clean(repo, wt).await.ok();
1932            if in_prep {
1933                self.rollback_prep(repo).await;
1934            }
1935            self.state.status = RunStatus::Blocked;
1936            self.state.event(
1937                "setup",
1938                format!("worktree setup failed in {}: {e:#}", wt.display()),
1939            );
1940            self.state.save()?;
1941            return Err(e);
1942        }
1943        Ok(())
1944    }
1945
1946    /// Undo everything `prep` created, so a resume re-runs it whole.
1947    async fn rollback_prep(&mut self, repo: &Path) {
1948        let root = self.state.worktree_root();
1949        for c in std::mem::take(&mut self.state.candidates) {
1950            if matches!(
1951                git::worktree_remove_clean(repo, &c.worktree).await,
1952                Ok(true)
1953            ) {
1954                git::branch_delete(repo, &c.branch).await.ok();
1955            } else {
1956                self.state.candidates.push(c);
1957            }
1958        }
1959        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1960        for j in 1..=seats {
1961            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1962                .await
1963                .ok();
1964        }
1965        for k in 1..=self.state.config.graph.advisors {
1966            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1967                .await
1968                .ok();
1969        }
1970    }
1971
1972    async fn prep(&mut self) -> Result<()> {
1973        if let Some(reason) = &self.state.prep_failure {
1974            bail!(
1975                "candidate preparation previously failed: {reason}; correct the source before retrying"
1976            );
1977        }
1978        if self
1979            .state
1980            .prep_complete
1981            .unwrap_or(!self.state.candidates.is_empty())
1982        {
1983            return Ok(());
1984        }
1985        let repo = self.state.repo.clone();
1986        // Includes rollback, config, refs, checkout creation and setup. No
1987        // repository mutation may precede the real shared lock acquisition.
1988        let _coordination = self.coordinate(&repo).await?;
1989        let base = self.state.base_commit.clone();
1990        let plan = self.state.sources.plan(&repo, &base).await?;
1991        self.rollback_prep(&repo).await;
1992        if !self.state.candidates.is_empty() {
1993            bail!(
1994                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1995            );
1996        }
1997        self.state.prep_complete = Some(false);
1998        self.state.status = RunStatus::Prep;
1999        self.state.save()?;
2000        let repo = self.state.repo.clone();
2001        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2002        let root = self.state.worktree_root();
2003        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2004
2005        // The hook is the write-time half of the blindness contract; the
2006        // presentation filter in `blind` is the half that cannot be bypassed.
2007        let hooks_dir = self.state.dir().join("hooks");
2008        if self.state.config.blind.commit_msg_hook {
2009            std::fs::create_dir_all(&hooks_dir)
2010                .with_context(|| format!("create {}", hooks_dir.display()))?;
2011            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2012            let path = hooks_dir.join("commit-msg");
2013            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2014            make_executable(&path)?;
2015            // Ref-counted rather than a plain idempotent set: with more than
2016            // one run able to be in flight in the same repository at once
2017            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2018            // true?" check cannot tell "another run of mine still needs
2019            // this" from "nobody does", and the run that happens to finish
2020            // first would disable the hook out from under a sibling still
2021            // relying on it.
2022            git::acquire_worktree_config(&repo).await?;
2023            self.state.enabled_worktree_config = true;
2024        }
2025
2026        // `[worktree] setup` hides its products per worktree, which needs the
2027        // same per-worktree config the hook does. Held until fold, like it.
2028        ensure_setup_config(&mut self.state, &repo).await?;
2029
2030        for (index, (spec, label)) in self
2031            .roles
2032            .implementers
2033            .clone()
2034            .into_iter()
2035            .zip(labels)
2036            .enumerate()
2037        {
2038            let branch = self.state.branch_for(label);
2039            let worktree = root.join(format!("cand-{label}"));
2040            // Persist intent before Git can leave a partially created tree.
2041            // Candidate registration follows setup, so it cannot inventory
2042            // crashes or refusals during creation itself.
2043            self.state.event(
2044                "prep-intent",
2045                serde_json::json!({
2046                    "branch": branch,
2047                    "worktree": worktree,
2048                    "start": start,
2049                    "picks": plan.picks,
2050                })
2051                .to_string(),
2052            );
2053            self.state.save()?;
2054            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2055            if self.state.config.blind.commit_msg_hook {
2056                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2057            }
2058            git::local_exclude(&worktree, "/.magi/").await?;
2059            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2060                // Nothing of this prep may survive: a resume must start over,
2061                // not find half the candidates and skip the rest.
2062                git::worktree_remove_clean(&repo, &worktree).await.ok();
2063                git::branch_delete(&repo, &branch).await.ok();
2064                self.rollback_prep(&repo).await;
2065                self.state.status = RunStatus::Blocked;
2066                self.state
2067                    .event("prep", format!("worktree setup failed: {e:#}"));
2068                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2069                self.state.save()?;
2070                return Err(e);
2071            }
2072            self.state.candidates.push(Candidate {
2073                index,
2074                label,
2075                agent: spec.id.clone(),
2076                branch,
2077                worktree,
2078                summary: String::new(),
2079                stat: String::new(),
2080                files: 0,
2081                commits: 0,
2082                empty: false,
2083                failed: None,
2084                verified_noop: None,
2085                duration_ms: 0,
2086                folded: false,
2087            });
2088            self.state.save()?;
2089            for pick in &plan.picks {
2090                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2091                {
2092                    self.state.status = RunStatus::Blocked;
2093                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2094                    self.state
2095                        .event("prep", format!("cannot apply referenced commit: {e}"));
2096                    self.state.save()?;
2097                    return Err(e);
2098                }
2099            }
2100        }
2101
2102        for j in 1..=self.roles.judges.len() {
2103            let wt = root.join(format!("judge-{j}"));
2104            if !wt.exists() {
2105                git::worktree_add_detached(&repo, &wt, &base).await?;
2106                self.setup_seat_worktree(&repo, &wt, true).await?;
2107            }
2108        }
2109
2110        // Disposable, detached checkouts for the design-deliberation stage's
2111        // advisor seats — the same shape as the judges' above, at the same
2112        // base commit, since advisors also only ever read. Sized off the
2113        // configured count directly rather than a resolved roster: unlike
2114        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2115        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2116        // `prep` has no `ResolvedRoles` field to read a count from here.
2117        if self.state.config.graph.advise {
2118            for k in 1..=self.state.config.graph.advisors {
2119                let wt = root.join(format!("advisor-{k}"));
2120                if !wt.exists() {
2121                    git::worktree_add_detached(&repo, &wt, &base).await?;
2122                    self.setup_seat_worktree(&repo, &wt, true).await?;
2123                }
2124            }
2125        }
2126
2127        // A judge cannot tell it is looking at its own patch — the seats keep
2128        // separate conversations — but a panel that shares agents with the
2129        // field is less independent than it looks, and that is worth saying out
2130        // loud once per run rather than leaving it in the config.
2131        let authors: Vec<&str> = self
2132            .roles
2133            .implementers
2134            .iter()
2135            .map(|a| a.id.as_str())
2136            .collect();
2137        let overlap: Vec<String> = self
2138            .roles
2139            .judges
2140            .iter()
2141            .enumerate()
2142            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2143            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2144            .collect();
2145        if !overlap.is_empty() {
2146            let note = format!(
2147                "{} also authored a candidate; blind, but the panel is less \
2148                 independent than {} distinct agents would be",
2149                overlap.join(", "),
2150                self.roles.judges.len()
2151            );
2152            self.state.event("prep", note);
2153        }
2154
2155        self.state.event(
2156            "prep",
2157            format!(
2158                "{} candidates, {} judges, base {} ({})",
2159                self.state.candidates.len(),
2160                self.roles.judges.len(),
2161                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2162                self.state.base_branch
2163            ),
2164        );
2165        self.state.prep_complete = Some(true);
2166        self.state.status = RunStatus::Implementing;
2167        self.state.save()?;
2168        Ok(())
2169    }
2170
2171    // -------------------------------------------------------------- advise
2172
2173    /// The design-deliberation stage: independent, read-only advisor seats
2174    /// each sketch a design before any implementer touches the repository,
2175    /// and (when at least one produced a usable proposal) a synthesis seat
2176    /// blends them into a brief `implement` carries in every candidate's
2177    /// prompt.
2178    ///
2179    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2180    /// advisors` is the proposal count. Everything here is best-effort and
2181    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2182    /// that cannot reach quota, or a synthesis seat that produced nothing
2183    /// usable all leave `implement` exactly as it was before this stage
2184    /// existed — the task instruction alone — rather than failing the whole
2185    /// competition over an enrichment stage. Every outcome is still recorded
2186    /// as an event, so a run that got nothing from this stage says why.
2187    ///
2188    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2189    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2190    /// resumed run whose stage failed would re-run it, and re-spend the
2191    /// agent calls, on every reentry before `implement`.
2192    ///
2193    /// Also skipped once any candidate shows implementation progress — the
2194    /// exact predicate `implement` itself uses to decide a candidate is no
2195    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2196    /// is not enough: a run created by an older binary that predates this
2197    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2198    /// an already-`Implementing`-or-later run under this build would
2199    /// otherwise walk straight back through `prep` (a no-op once candidates
2200    /// exist) into this node and spawn every advisor seat against worktrees
2201    /// `prep` never recreated — after implementation has already started,
2202    /// which is exactly the invariant this stage exists to guarantee.
2203    async fn advise(&mut self) -> Result<()> {
2204        let implement_untouched = self
2205            .state
2206            .candidates
2207            .iter()
2208            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2209        if !self.state.config.graph.advise || self.state.advise_attempted {
2210            return Ok(());
2211        }
2212        if !implement_untouched {
2213            self.state.event(
2214                "advise",
2215                "skipping the design-deliberation stage: at least one \
2216                 candidate already shows implementation progress, so this \
2217                 run is past the point the stage exists to run before"
2218                    .to_owned(),
2219            );
2220            self.state.advise_attempted = true;
2221            self.state.save()?;
2222            return Ok(());
2223        }
2224        let run_id = self.state.id.clone();
2225        let prompts = self.state.config.prompts.clone();
2226        let instruction = self.state.instruction.clone();
2227        let language = self.state.config.graph.language.clone();
2228        let root = self.state.worktree_root();
2229        let n = self.state.config.graph.advisors;
2230        let where_recorded = self.state.dir().join("run.json");
2231
2232        let seats = match self.state.config.advisors() {
2233            Ok(seats) if !seats.is_empty() => seats,
2234            Ok(_) => {
2235                self.state.event(
2236                    "advise",
2237                    format!(
2238                        "[graph] advisors is 0; skipping the design-deliberation \
2239                         stage and continuing without a synthesis brief (see {})",
2240                        where_recorded.display()
2241                    ),
2242                );
2243                self.state.advise_attempted = true;
2244                self.state.save()?;
2245                return Ok(());
2246            }
2247            Err(e) => {
2248                self.state.event(
2249                    "advise",
2250                    format!(
2251                        "could not resolve advisor seats ({e:#}); continuing \
2252                         without a design-deliberation brief (see {})",
2253                        where_recorded.display()
2254                    ),
2255                );
2256                self.state.advise_attempted = true;
2257                self.state.save()?;
2258                return Ok(());
2259            }
2260        };
2261
2262        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2263        let artifacts = agent::artifacts_dir(&self.state.dir());
2264        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2265
2266        let mut jobs = Vec::new();
2267        for (i, spec) in seats.iter().cloned().enumerate() {
2268            let seat_key = format!("advisor-{}", i + 1);
2269            let seat = self.seat(&seat_key, &spec.id);
2270            jobs.push(SeatJob {
2271                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2272                spec,
2273                seat,
2274                cwd: worktrees[i % worktrees.len()].clone(),
2275                timeout,
2276                allow_write: false,
2277                sessions: false,
2278                artifacts: artifacts.clone(),
2279                stem: seat_key,
2280                handover: None,
2281            });
2282        }
2283
2284        self.state.event(
2285            "advise",
2286            format!(
2287                "{} advisor seat(s) sketching a design in parallel",
2288                jobs.len()
2289            ),
2290        );
2291        let mut quota_losses = Vec::new();
2292        let cache = self.state.config.cache_dir();
2293        let ctx = WaveCtx {
2294            carry_seats: false,
2295            pauses: Some((&self.pause, &self.interrupt)),
2296            run: &run_id,
2297            node: "advise",
2298            prompts: &prompts,
2299            cache: cache.as_deref(),
2300            round: None,
2301        };
2302        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2303        let results = ask_json_wave::<Proposal>(
2304            jobs,
2305            Arc::clone(&self.sem),
2306            self.state.config.graph.retries,
2307            &advisor_roster,
2308            &ctx,
2309            &mut quota_losses,
2310            &mut self.state,
2311            &|p: &Proposal| p.validate(),
2312        )
2313        .await;
2314        if self.state.resource_block.is_some() {
2315            return self.stop_for_resource();
2316        }
2317        self.state.quota.extend(quota_losses);
2318
2319        let mut records = Vec::with_capacity(results.len());
2320        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2321            let agent_id = seat.agent.clone();
2322            self.state.seats.insert(seat.key.clone(), seat);
2323            match res {
2324                Ok((proposal, out)) => {
2325                    self.state
2326                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2327                    records.push(advise::AdvisorRecord::proposed(
2328                        i + 1,
2329                        agent_id,
2330                        proposal,
2331                        out.duration_ms,
2332                    ));
2333                }
2334                Err(e) => {
2335                    self.state.event(
2336                        "advise",
2337                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2338                    );
2339                    records.push(advise::AdvisorRecord::failed(
2340                        i + 1,
2341                        agent_id,
2342                        e.to_string(),
2343                    ));
2344                }
2345            }
2346        }
2347
2348        let mut advice = advise::Advice {
2349            records,
2350            synthesis: None,
2351        };
2352        if advice.proposals().is_empty() {
2353            self.state.event(
2354                "advise",
2355                "no advisor produced a usable proposal; continuing without a \
2356                 synthesis brief"
2357                    .to_owned(),
2358            );
2359        } else {
2360            match self
2361                .synthesize_brief(
2362                    &advice,
2363                    &instruction,
2364                    &language,
2365                    &worktrees[0],
2366                    &artifacts,
2367                    &run_id,
2368                    &prompts,
2369                    cache.as_deref(),
2370                )
2371                .await
2372            {
2373                Ok(Some(text)) => {
2374                    self.state.event(
2375                        "advise",
2376                        "synthesized a design brief for the implementer".to_owned(),
2377                    );
2378                    advice.synthesis = Some(text);
2379                }
2380                Ok(None) => {
2381                    self.state.event(
2382                        "advise",
2383                        "the synthesis seat produced nothing usable; continuing \
2384                         without a design brief"
2385                            .to_owned(),
2386                    );
2387                }
2388                Err(e) => {
2389                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2390                        return Err(e);
2391                    }
2392                    self.state.event(
2393                        "advise",
2394                        format!("could not synthesize a design brief: {e:#}"),
2395                    );
2396                }
2397            }
2398        }
2399        advise::apply_reflection(&mut advice);
2400
2401        self.state.advice = Some(advice);
2402        self.state.advise_attempted = true;
2403        self.state.save()?;
2404        Ok(())
2405    }
2406
2407    /// The synthesis seat: reads every advisor's proposal and blends them
2408    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2409    /// out of [`Runner::advise`] only for readability — it is not called
2410    /// anywhere else.
2411    ///
2412    /// Picked the same way [`crate::talk`]'s standing conversation and
2413    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2414    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2415    /// order (a claude seat, else the first runnable agent in roster order)
2416    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2417    /// in [`crate::config`] for why a dedicated field exists here at all.
2418    #[allow(clippy::too_many_arguments)]
2419    async fn synthesize_brief(
2420        &mut self,
2421        advice: &advise::Advice,
2422        instruction: &str,
2423        language: &str,
2424        cwd: &Path,
2425        artifacts: &Path,
2426        run_id: &str,
2427        prompts: &Prompts,
2428        cache: Option<&Path>,
2429    ) -> Result<Option<String>> {
2430        let chain = agent::pick_chain(
2431            &self.state.config.agents,
2432            self.state.config.roles.synthesizer.as_ref(),
2433            &agent::installed,
2434            "synthesizer",
2435        )?;
2436        let proposals = advice.proposals();
2437        let mut prompt = prompt::with_overlay(
2438            prompt::synthesize_brief(instruction, &proposals, language),
2439            prompts.overlay("advise"),
2440        );
2441        if cache.is_some() {
2442            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2443            // below — see `prompt::build_cache_note`'s doc for why telling a
2444            // read-only seat to build through the shared cache is exactly how
2445            // a sandbox's write refusal gets misread as a defect.
2446            prompt.push('\n');
2447            prompt.push_str(&prompt::build_cache_note("advise", false));
2448        }
2449        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2450        // Each id is tried once, in order; a quota hit, error or unusable
2451        // answer moves to the next. The seat is single-turn (`sessions:
2452        // false`) and the prompt is the whole context, so a fallback agent
2453        // needs nothing carried over.
2454        let mut last = None;
2455        for (n, spec) in chain.iter().enumerate() {
2456            if n > 0 {
2457                self.state
2458                    .event("advise", format!("synthesis falling back to {}", spec.id));
2459            }
2460            let mut seat = self.seat("advise-synthesis", &spec.id);
2461            let wait = self.repository_wait();
2462            let outcome = wait
2463                .scope(agent::invoke(
2464                    spec,
2465                    &mut seat,
2466                    &Invocation {
2467                        coordination: self.state.config.coordination.as_ref(),
2468                        fetched_base: None,
2469                        cwd,
2470                        prompt: &prompt,
2471                        timeout,
2472                        allow_write: false,
2473                        sessions: false,
2474                        artifacts,
2475                        stem: &if n == 0 {
2476                            "advise-synthesis".to_owned()
2477                        } else {
2478                            format!("advise-synthesis-{}", spec.id)
2479                        },
2480                        run: run_id,
2481                        node: "advise",
2482                        cache_dir: None,
2483                        attachments: &[],
2484                        writable: &[],
2485                    },
2486                ))
2487                .await;
2488            if outcome.is_ok() {
2489                self.state.seats.insert(seat.key.clone(), seat);
2490            }
2491            let advance = agent::chain_advances(&outcome);
2492            last = Some(outcome);
2493            if !advance {
2494                break;
2495            }
2496        }
2497        // Exhausted: the last attempt's result is what a single failed seat
2498        // would have produced.
2499        let out = last.expect("a chain holds at least one agent")?;
2500        if !out.usable() {
2501            return Ok(None);
2502        }
2503        let text =
2504            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2505        Ok((!text.trim().is_empty()).then_some(text))
2506    }
2507
2508    // ----------------------------------------------------------- implement
2509
2510    async fn implement(&mut self) -> Result<()> {
2511        self.persist_candidates().await?;
2512        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2513        // agent files with `magi task add` name the run that paid for it. The
2514        // prompt overlay is cloned alongside it because the waves borrow it
2515        // while `self` is mutably borrowed by the node's own bookkeeping.
2516        let run_id = self.state.id.clone();
2517        let prompts = self.state.config.prompts.clone();
2518        let todo: Vec<usize> = self
2519            .state
2520            .candidates
2521            .iter()
2522            .enumerate()
2523            .filter(|(_, c)| {
2524                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2525                    && c.failed.is_none()
2526                    && !c.empty
2527            })
2528            .map(|(i, _)| i)
2529            .collect();
2530        if todo.is_empty() {
2531            return self.after_implement();
2532        }
2533        self.state.status = RunStatus::Implementing;
2534
2535        let language = self.state.config.graph.language.clone();
2536        let github_language = self.state.config.graph.github_language.clone();
2537        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2538        let sessions = self.state.config.graph.sessions;
2539        let artifacts = agent::artifacts_dir(&self.state.dir());
2540        // The design-deliberation stage's blended brief, when `advise` found
2541        // one — carried into every implementer's prompt the same way
2542        // regardless of which candidate it is.
2543        let brief = self
2544            .state
2545            .advice
2546            .as_ref()
2547            .and_then(|a| a.synthesis.as_deref())
2548            .map(str::to_owned);
2549        let attachments = self.state.attachments.clone();
2550
2551        let mut jobs = Vec::new();
2552        for &i in &todo {
2553            let (index, label, worktree) = {
2554                let c = &self.state.candidates[i];
2555                (c.index, c.label, c.worktree.clone())
2556            };
2557            let spec = self.roles.implementers[index].clone();
2558            let seat_key = format!("impl-{label}");
2559            let seat = self.seat(&seat_key, &spec.id);
2560            let instruction = seeded_instruction(&self.state);
2561            jobs.push(SeatJob {
2562                spec,
2563                seat,
2564                prompt: prompt::implement_in(
2565                    &instruction,
2566                    &worktree.to_string_lossy(),
2567                    &language,
2568                    &github_language,
2569                    brief.as_deref(),
2570                    &attachments,
2571                ),
2572                cwd: worktree,
2573                timeout,
2574                allow_write: true,
2575                sessions,
2576                artifacts: artifacts.clone(),
2577                stem: format!("impl-{label}"),
2578                handover: None,
2579            });
2580        }
2581
2582        self.state.event(
2583            "implement",
2584            format!("{} candidates in parallel", jobs.len()),
2585        );
2586        // Kept so a seat whose CLI hung up can be asked again from the same
2587        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2588        // can update a seat's own entry once a fallback agent takes it over —
2589        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2590        // whichever agent actually answered, not the one that quota'd out.
2591        let mut sent = jobs.clone();
2592        let cache = self.state.config.cache_dir();
2593        let ctx = WaveCtx {
2594            carry_seats: false,
2595            pauses: Some((&self.pause, &self.interrupt)),
2596            run: &run_id,
2597            node: "implement",
2598            prompts: &prompts,
2599            cache: cache.as_deref(),
2600            round: None,
2601        };
2602        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2603        if !wave_blocked(&results) {
2604            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2605                .await;
2606            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2607                .await?;
2608            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2609                .await;
2610        }
2611        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2612
2613        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2614            && results
2615                .iter()
2616                .any(|(_, _, out)| FailClass::of(out).is_some());
2617        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2618            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2619                // Executed turns interrupted by disk pressure already carry
2620                // a preservation barrier from wave. Admission-only waits do
2621                // not. Keep the interrupted seat's session for recovery too.
2622                if self
2623                    .state
2624                    .candidate_preservation
2625                    .contains_key(&self.state.candidates[i].label)
2626                {
2627                    self.state.seats.insert(seat.key.clone(), seat);
2628                }
2629                continue;
2630            }
2631            let seat_key = seat.key.clone();
2632            // A quota fallback (`resume_seat_handovers`) may have handed this
2633            // seat to a different agent than the one `prep` recorded on the
2634            // candidate; the stats tables and any later fixer-defaults-to-
2635            // winner's-author lookup must credit whoever actually answered —
2636            // unless every fallback also quota'd out, in which case nobody
2637            // actually answered and crediting the last agent tried would
2638            // erase every earlier agent's own quota loss from the stats
2639            // tables instead of just this one seat's.
2640            let agent = seat.agent.clone();
2641            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2642            self.state.seats.insert(seat.key.clone(), seat);
2643            let label = self.state.candidates[i].label;
2644            self.state.interrupted_implementations.remove(&label);
2645
2646            let (summary, duration, failed, verified_claim) = match out {
2647                AgentOutcome::Ok(o) => {
2648                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2649                    let failed = (!o.usable()).then(|| {
2650                        if o.timed_out {
2651                            "agent timed out".to_owned()
2652                        } else {
2653                            format!(
2654                                "agent exited with {}",
2655                                crate::notices::exit_status(o.exit_code)
2656                            )
2657                        }
2658                    });
2659                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2660                    (text, o.duration_ms, failed, verified_claim)
2661                }
2662                // Left un-resumed by `resume_undelivered` (a dirty tree
2663                // already rescues the work, or there was no session left to
2664                // resume into) — reported like the ordinary failure it is,
2665                // never as if `o.text` (the CLI's raw error JSON) were an
2666                // answer.
2667                AgentOutcome::Dropped(o) => {
2668                    let why = o
2669                        .dropped
2670                        .as_ref()
2671                        .map(|d| d.why.as_str())
2672                        .unwrap_or("the CLI ended the stream without delivering its answer");
2673                    (
2674                        String::new(),
2675                        o.duration_ms,
2676                        Some(format!("the CLI dropped the stream ({why})")),
2677                        None,
2678                    )
2679                }
2680                AgentOutcome::Quota(o) => {
2681                    self.state.quota.push(QuotaLoss {
2682                        seat: seat_key,
2683                        node: "implement".to_owned(),
2684                        at: Timestamp::now(),
2685                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2686                    });
2687                    (
2688                        String::new(),
2689                        o.duration_ms,
2690                        Some("rate limited (quota); produced no change".to_owned()),
2691                        None,
2692                    )
2693                }
2694                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2695                    (String::new(), 0, Some(e), None)
2696                }
2697            };
2698
2699            // Save every completed turn before any fallible Git operation. A failed
2700            // rescue must not erase the reply or prevent collection of other seats.
2701            let c = &mut self.state.candidates[i];
2702            if !exhausted_the_fallback_chain {
2703                c.agent = agent;
2704            }
2705            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2706            c.duration_ms = duration;
2707            c.failed = failed;
2708            c.verified_noop = verified_claim;
2709            self.state
2710                .candidate_preservation
2711                .insert(label, "awaiting persistence".to_owned());
2712        }
2713        self.state.save()?;
2714        self.persist_candidates().await?;
2715
2716        if resource_blocked {
2717            return self.stop_for_resource();
2718        }
2719        if selected_failed {
2720            self.state.status = RunStatus::Blocked;
2721            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2722            self.state.save()?;
2723            bail!(
2724                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2725            );
2726        }
2727        self.after_implement()
2728    }
2729
2730    /// Retry only persistence after a completed implementation turn.
2731    async fn persist_candidates(&mut self) -> Result<()> {
2732        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2733        for label in labels {
2734            let i = self
2735                .state
2736                .candidates
2737                .iter()
2738                .position(|c| c.label == label)
2739                .context("pending candidate is missing")?;
2740            let c = self.state.candidates[i].clone();
2741            let interrupted = self.state.interrupted_implementations.contains(&label);
2742            let result = async {
2743                let wait = crate::repo_lock::WaitPolicy::new(
2744                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2745                    || false,
2746                );
2747                let _coordination =
2748                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2749                // Best effort evidence requires no index writes. A successful snapshot
2750                // never authorizes removal of a tree whose rescue failed.
2751                if let Err(e) = git::snapshot_uncommitted(
2752                    &c.worktree,
2753                    &self
2754                        .state
2755                        .dir()
2756                        .join("artifacts")
2757                        .join(format!("candidate-{label}-recovery")),
2758                )
2759                .await
2760                {
2761                    self.state.event(
2762                        "preserve",
2763                        format!("candidate {label} snapshot incomplete: {e:#}"),
2764                    );
2765                }
2766                let rescued = git::rescue_commit(
2767                    &c.worktree,
2768                    &format!("magi: candidate {label} (uncommitted work)"),
2769                )
2770                .await?;
2771                self.state.note_withheld("implement", &rescued.withheld);
2772                let base = &self.state.base_commit;
2773                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2774                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2775                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2776                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2777                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2778                if c.empty && commits > 0 && !patch.trim().is_empty()
2779                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2780                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2781                        || self.state.tally.is_some())
2782                {
2783                    // Keep the old classification and pending marker so every
2784                    // retry stops too. Retain decisions as recovery evidence;
2785                    // only a supported recovery operation may replace them.
2786                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2787                }
2788                let c = &mut self.state.candidates[i];
2789                c.commits = commits;
2790                c.stat = stat;
2791                c.files = files;
2792                let no_changes = commits == 0 || patch.trim().is_empty();
2793                c.empty = no_changes && !interrupted;
2794                if interrupted && no_changes {
2795                    c.failed = None;
2796                    c.verified_noop = None;
2797                }
2798                if !c.empty {
2799                    c.failed = None;
2800                    c.verified_noop = None;
2801                }
2802                self.state.event(
2803                    "preserve",
2804                    format!("candidate {label}: {files} files, {commits} commits"),
2805                );
2806                Ok::<_, anyhow::Error>(())
2807            }
2808            .await;
2809            match result {
2810                Ok(()) => {
2811                    self.state.candidate_preservation.remove(&label);
2812                    if !interrupted || self.state.candidates[i].files > 0 {
2813                        self.state.interrupted_implementations.remove(&label);
2814                    }
2815                }
2816                Err(e) => {
2817                    let reason = format!("candidate {label} preservation failed: {e:#}");
2818                    self.state.event("preserve", &reason);
2819                    self.state.candidate_preservation.insert(label, reason);
2820                }
2821            }
2822            if self.state.candidate_preservation.is_empty()
2823                && self.state.status == RunStatus::Blocked
2824                && self.state.candidates.iter().all(|c| c.failed.is_none())
2825            {
2826                // Persist the recovered phase with the cleared pending marker;
2827                // a crash between saves must not turn this into a selected-seat
2828                // failure hold on the next resume.
2829                self.state.status = RunStatus::Implementing;
2830                self.state.event(
2831                    "preserve",
2832                    "candidate persistence recovered; implementation can continue",
2833                );
2834            }
2835            self.state.save()?;
2836        }
2837        if !self.state.candidate_preservation.is_empty() {
2838            self.state.status = RunStatus::Blocked;
2839            self.state.save()?;
2840            bail!(
2841                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2842            );
2843        }
2844        Ok(())
2845    }
2846
2847    /// Ask again, once, for work a CLI did and then failed to hand over.
2848    ///
2849    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2850    /// status with an empty response and a usage report showing output tokens,
2851    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2852    /// seven minutes and 14,267 output tokens that arrived as an empty
2853    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2854    ///
2855    /// Two conditions, and both matter:
2856    ///
2857    /// - **Only when the tree is untouched.** Often the agent has already
2858    ///   written its files and only the closing message was lost; the rescue
2859    ///   commit below picks that up and there is nothing to ask for. Re-asking
2860    ///   then would pay for a second implementation of work already on disk.
2861    /// - **Once.** A CLI that drops one stream can drop the next, and this
2862    ///   node is the most expensive in the graph.
2863    ///
2864    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2865    /// dropped reply still carried its `conversation_id`, so the seat is asked
2866    /// to finish what it was doing rather than sent the whole task again. It
2867    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2868    /// node's - for the same reason a re-ranked judge does: restating finished
2869    /// work is not the work.
2870    ///
2871    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2872    /// same way until it resets, while an abandoned conversation is still
2873    /// there to be picked up.
2874    async fn resume_undelivered(
2875        &mut self,
2876        results: &mut [(usize, SeatState, AgentOutcome)],
2877        sent: &[SeatJob],
2878        prompts: &Prompts,
2879        run_id: &str,
2880    ) {
2881        for (wi, seat, out) in results.iter_mut() {
2882            let Some(dropped) = (match &*out {
2883                AgentOutcome::Dropped(o) => o.dropped.clone(),
2884                _ => None,
2885            }) else {
2886                continue;
2887            };
2888            let Some(job) = sent.get(*wi) else { continue };
2889            // Already on disk? Then only the closing message was lost.
2890            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2891                self.state.event(
2892                    "implement",
2893                    format!(
2894                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2895                         work is in the tree",
2896                        seat.key, dropped.output_tokens, dropped.why
2897                    ),
2898                );
2899                continue;
2900            }
2901            // The re-ask only makes sense as a resume: `resume_after_drop`
2902            // says nothing about the task, trusting the seat to still hold it.
2903            // Without a session to resume — sessions disabled, or this CLI's
2904            // drop shape happened not to carry a session id — that prompt
2905            // would open a brand-new conversation with no context at all,
2906            // which is worse than leaving this as the ordinary failure it
2907            // already is.
2908            if !has_context(&job.spec, seat, job.sessions) {
2909                self.state.event(
2910                    "implement",
2911                    format!(
2912                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2913                         is no session left to resume",
2914                        seat.key, dropped.output_tokens, dropped.why
2915                    ),
2916                );
2917                continue;
2918            }
2919            self.state.event(
2920                "implement",
2921                format!(
2922                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2923                     conversation",
2924                    seat.key, dropped.output_tokens, dropped.why
2925                ),
2926            );
2927            let mut retry = job.clone();
2928            retry.seat = seat.clone();
2929            retry.prompt = prompt::resume_after_drop(&dropped.why);
2930            retry.timeout = retry_budget(job.timeout, true);
2931            retry.stem = format!("{}-resume", job.stem);
2932            let cache = self.state.config.cache_dir();
2933            let ctx = WaveCtx {
2934                carry_seats: false,
2935                pauses: Some((&self.pause, &self.interrupt)),
2936                run: run_id,
2937                node: "implement",
2938                prompts,
2939                cache: cache.as_deref(),
2940                round: None,
2941            };
2942            let (resumed_seat, resumed) =
2943                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2944            *seat = resumed_seat;
2945            *out = resumed;
2946        }
2947    }
2948
2949    /// Fall an implement seat through to the next untried agent in the
2950    /// implementer roster when it lost to quota — or, since the handover was
2951    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2952    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2953    /// quota path itself being unchanged — instead of leaving the
2954    /// seat's loss final the moment one agent's account runs dry.
2955    ///
2956    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2957    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2958    /// truncates to the single slot rotation picked, so a solo task whose one
2959    /// implementer hits quota mid-run used to have nothing else to try. This
2960    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2961    /// unrotated roster — which is the only place the *other* candidates in
2962    /// the machine's roster still exist once `implementers` has been cut down
2963    /// to size.
2964    ///
2965    /// Walks forward from just past the seat's own original position in the
2966    /// roster, never wrapping back to the front: a later candidate slot (say
2967    /// `beta`, the roster's second entry) must fall through to the *next*
2968    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2969    /// almost certainly a different candidate's own agent already — and once
2970    /// the roster's tail is exhausted there is nothing left to fall through
2971    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2972    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2973    /// let this retry that id forever. The loop keeps falling through until
2974    /// an attempt lands something other than `Quota` or the roster's tail
2975    /// runs out of untried ids, at which point the seat is left exactly as
2976    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2977    /// `QuotaLoss` recorded, the candidate failed/empty.
2978    ///
2979    /// `sent` is taken mutably and updated with the fallback agent's spec:
2980    /// `resume_unconfirmed_commands`, which runs after this and also reads
2981    /// `sent`, must see whichever agent actually ended up answering the seat
2982    /// — reading the stale, original spec there would check session
2983    /// eligibility against the wrong CLI and could hand a fallback agent's
2984    /// session id to the agent that just lost the seat to quota.
2985    ///
2986    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2987    /// — `self.seat` only reuses state when the agent id is unchanged, so
2988    /// handing it a different id already gets this for free. Reusing the old
2989    /// seat would resume a different CLI's session as if it were a
2990    /// continuation of this one.
2991    ///
2992    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2993    /// a quota loss cuts an agent off mid-turn, so anything already in the
2994    /// tree is unfinished work, not a completed candidate a re-ask would pay
2995    /// for twice. A dirty tree is rescued into a commit first (the same
2996    /// neutral-identity rescue `implement`'s own outcome loop gives every
2997    /// candidate) so the next agent starts clean.
2998    ///
2999    /// The new agent gets the implementer's full prompt and full
3000    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3001    /// it has no session and no context, and is implementing the task from
3002    /// nothing, unlike a resumed drop which is only restating work already
3003    /// done.
3004    ///
3005    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3006    /// `implement` event, never into `self.state.quota` — that is what
3007    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3008    /// go unspent, and a seat that ultimately recovered on its second or
3009    /// third agent is not the stalled panel that check exists to catch. Only
3010    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3011    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3012    /// outcome loop already has — this helper never pushes to it itself.
3013    async fn resume_seat_handovers(
3014        &mut self,
3015        results: &mut [(usize, SeatState, AgentOutcome)],
3016        sent: &mut [SeatJob],
3017        prompts: &Prompts,
3018        run_id: &str,
3019    ) -> Result<()> {
3020        let instruction = seeded_instruction(&self.state);
3021        let language = self.state.config.graph.language.clone();
3022        let github_language = self.state.config.graph.github_language.clone();
3023        let brief = self
3024            .state
3025            .advice
3026            .as_ref()
3027            .and_then(|a| a.synthesis.as_deref())
3028            .map(str::to_owned);
3029        let attachments = self.state.attachments.clone();
3030        for (wi, seat, out) in results.iter_mut() {
3031            // Who holds the other candidate seats of this wave right now
3032            // (earlier handovers already written back to `sent`).
3033            let others: BTreeSet<String> = sent
3034                .iter()
3035                .enumerate()
3036                .filter(|(j, _)| j != wi)
3037                .map(|(_, j)| j.spec.id.clone())
3038                .collect();
3039            let Some(job) = sent.get_mut(*wi) else {
3040                continue;
3041            };
3042            // Where the seat's own original agent sits in the roster — the
3043            // fallback walk starts just past here, never at the front, so a
3044            // later candidate slot's quota loss does not fall back onto an
3045            // earlier slot's own agent.
3046            let start = self
3047                .roles
3048                .implementer_roster
3049                .iter()
3050                .position(|s| s.id == job.spec.id)
3051                .unwrap_or(0);
3052            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3053            let mut fallback_attempt = 0usize;
3054            let mut prev: Option<FailClass> = None;
3055            while let Some(cur) = FailClass::of(&*out) {
3056                if !should_hand_over(prev.as_ref(), &cur) {
3057                    break;
3058                }
3059                let Some(next) =
3060                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3061                        .cloned()
3062                else {
3063                    break;
3064                };
3065                tried.insert(next.id.clone());
3066                fallback_attempt += 1;
3067
3068                let coordination = self.coordinate(&job.cwd).await?;
3069                if let Ok(r) = git::rescue_commit(
3070                    &job.cwd,
3071                    &format!(
3072                        "magi: candidate {} (uncommitted work before {} fallback)",
3073                        seat.key,
3074                        if cur == FailClass::Quota {
3075                            "quota"
3076                        } else {
3077                            "handover"
3078                        }
3079                    ),
3080                )
3081                .await
3082                {
3083                    self.state.note_withheld("implement", &r.withheld);
3084                }
3085                drop(coordination);
3086
3087                record_handover(
3088                    &mut self.state,
3089                    "implement",
3090                    &seat.key,
3091                    &seat.agent,
3092                    &next.id,
3093                    &cur,
3094                    &fail_reason(&*out),
3095                );
3096                prev = Some(cur.clone());
3097
3098                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3099                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3100                // Kept in sync on `sent` itself, not just the local retry: a
3101                // later helper (`resume_unconfirmed_commands`) reads `sent`
3102                // after this one returns and must see whichever agent is now
3103                // occupying the seat, not the one that just quota'd out —
3104                // otherwise it would judge session/continuation eligibility
3105                // by the wrong CLI and could resend a fallback's session id
3106                // to the agent that lost it the seat in the first place.
3107                job.spec = next.clone();
3108                let mut retry = job.clone();
3109                retry.seat = new_seat;
3110                retry.prompt = prompt::implement_in(
3111                    &instruction,
3112                    &job.cwd.to_string_lossy(),
3113                    &language,
3114                    &github_language,
3115                    brief.as_deref(),
3116                    &attachments,
3117                );
3118                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3119                let cache = self.state.config.cache_dir();
3120                let ctx = WaveCtx {
3121                    carry_seats: false,
3122                    pauses: Some((&self.pause, &self.interrupt)),
3123                    run: run_id,
3124                    node: "implement",
3125                    prompts,
3126                    cache: cache.as_deref(),
3127                    round: None,
3128                };
3129                let (fallback_seat, fallback_out) = run_one(
3130                    retry,
3131                    Arc::clone(&self.sem),
3132                    &ctx,
3133                    &mut self.state,
3134                    fallback_attempt,
3135                )
3136                .await;
3137                *seat = fallback_seat;
3138                *out = fallback_out;
3139            }
3140        }
3141        Ok(())
3142    }
3143
3144    /// Ask an implement seat's own CLI to confirm what it started, once, when
3145    /// its reply reported a command whose completion status it never
3146    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3147    /// that does and does not mean.
3148    ///
3149    /// The completion contract this task asks for, extended to `implement`
3150    /// with the same signal `continue_fix_report` reads for the fixer,
3151    /// rather than a keyword search over the reply or a hard requirement on
3152    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3153    /// a candidate's CLI turn ended cleanly while a test run it had started
3154    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3155    /// named in it at all is untouched by this: `commands` is empty, so
3156    /// there is nothing to be unconfirmed.
3157    ///
3158    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3159    /// this is not about recovering edits that might already be on disk, it
3160    /// is about a result the seat itself never vouched for, which resuming
3161    /// asks for regardless of what the tree already holds. Bounded to one
3162    /// attempt for the same reason `resume_undelivered` is — this is the
3163    /// most expensive node in the graph — and a seat that still cannot
3164    /// confirm on that attempt is left as whatever its (possibly still
3165    /// unconfirmed) reply says; this does not invent a new "failed" reason
3166    /// for a candidate that otherwise produced a real, committed change.
3167    async fn resume_unconfirmed_commands(
3168        &mut self,
3169        results: &mut [(usize, SeatState, AgentOutcome)],
3170        sent: &[SeatJob],
3171        prompts: &Prompts,
3172        run_id: &str,
3173    ) {
3174        for (wi, seat, out) in results.iter_mut() {
3175            let AgentOutcome::Ok(o) = &*out else {
3176                continue;
3177            };
3178            if !has_unconfirmed_command(&o.commands) {
3179                continue;
3180            }
3181            let Some(job) = sent.get(*wi) else { continue };
3182            if !has_context(&job.spec, seat, job.sessions) {
3183                self.state.event(
3184                    "implement",
3185                    format!(
3186                        "{}: the reply named a command whose own CLI never confirmed the exit \
3187                         status of, but there is no session left to resume",
3188                        seat.key
3189                    ),
3190                );
3191                continue;
3192            }
3193            self.state.event(
3194                "implement",
3195                format!(
3196                    "{}: the reply named a command whose own CLI never confirmed the exit \
3197                     status of; resuming the conversation",
3198                    seat.key
3199                ),
3200            );
3201            let mut retry = job.clone();
3202            retry.seat = seat.clone();
3203            retry.prompt = prompt::resume_incomplete(
3204                "a command in your last reply had no confirmed exit status",
3205            );
3206            retry.timeout = retry_budget(job.timeout, true);
3207            retry.stem = format!("{}-confirm", job.stem);
3208            let cache = self.state.config.cache_dir();
3209            let ctx = WaveCtx {
3210                carry_seats: false,
3211                pauses: Some((&self.pause, &self.interrupt)),
3212                run: run_id,
3213                node: "implement",
3214                prompts,
3215                cache: cache.as_deref(),
3216                round: None,
3217            };
3218            let (resumed_seat, resumed) =
3219                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3220            *seat = resumed_seat;
3221            *out = resumed;
3222        }
3223    }
3224
3225    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3226    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3227    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3228    /// that motivated this.
3229    ///
3230    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3231    /// own nudge loop already covers for judge/review/vote seats, and not a
3232    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3233    /// implement seats: here the CLI turn genuinely finished while the node's
3234    /// own work — the fixer's account of what it did — had not. Gated purely
3235    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3236    /// reply, never on any wording in it, so a fixer whose valid, first-try
3237    /// `FixReport` happens to mention having waited on a background test is
3238    /// never resumed — the `Ok(report)` branch at the call site returns
3239    /// before this is ever invoked.
3240    ///
3241    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3242    /// attempt ([`retry_budget`]), nothing attempted once the session is
3243    /// gone, and a quota hit ends the loop immediately rather than retrying a
3244    /// rate limit that fails the same way again.
3245    async fn continue_fix_report(
3246        &mut self,
3247        mut seat: SeatState,
3248        parse_err: String,
3249        job: &SeatJob,
3250        prompts: &Prompts,
3251        run_id: &str,
3252        round: usize,
3253    ) -> (
3254        SeatState,
3255        Option<FixReport>,
3256        Option<String>,
3257        ContinuationRecord,
3258    ) {
3259        let mut last_err = parse_err;
3260        let mut cumulative_wait_ms = 0u64;
3261        let mut attempts = 0usize;
3262        loop {
3263            if !has_context(&job.spec, &seat, job.sessions) {
3264                self.state.event(
3265                    "fix",
3266                    format!(
3267                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3268                         session left to resume into"
3269                    ),
3270                );
3271                let outcome = if attempts == 0 {
3272                    ContinuationOutcome::NoSession
3273                } else {
3274                    ContinuationOutcome::Exhausted
3275                };
3276                return (
3277                    seat,
3278                    None,
3279                    Some(format!("unparsable fix report: {last_err}")),
3280                    ContinuationRecord {
3281                        attempts,
3282                        cumulative_wait_ms,
3283                        outcome,
3284                    },
3285                );
3286            }
3287            if attempts >= MAX_FIX_CONTINUATIONS {
3288                self.state.event(
3289                    "fix",
3290                    format!(
3291                        "round {round}: fixer's reply still had no adoption report after \
3292                         {attempts} continuation(s) ({last_err}); giving up"
3293                    ),
3294                );
3295                return (
3296                    seat,
3297                    None,
3298                    Some(format!(
3299                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3300                    )),
3301                    ContinuationRecord {
3302                        attempts,
3303                        cumulative_wait_ms,
3304                        outcome: ContinuationOutcome::Exhausted,
3305                    },
3306                );
3307            }
3308            attempts += 1;
3309            self.state.event(
3310                "fix",
3311                format!(
3312                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3313                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3314                ),
3315            );
3316            let mut retry = job.clone();
3317            retry.seat = seat.clone();
3318            retry.prompt = prompt::resume_incomplete(&last_err);
3319            retry.timeout = retry_budget(job.timeout, true);
3320            retry.stem = format!("{}-continue{attempts}", job.stem);
3321            let cache = self.state.config.cache_dir();
3322            let ctx = WaveCtx {
3323                carry_seats: false,
3324                pauses: Some((&self.pause, &self.interrupt)),
3325                run: run_id,
3326                node: "fix",
3327                prompts,
3328                cache: cache.as_deref(),
3329                round: Some(round),
3330            };
3331            let (resumed_seat, resumed_out) = run_one(
3332                retry,
3333                Arc::clone(&self.sem),
3334                &ctx,
3335                &mut self.state,
3336                attempts,
3337            )
3338            .await;
3339            seat = resumed_seat;
3340            match resumed_out {
3341                AgentOutcome::Ok(o) => {
3342                    cumulative_wait_ms += o.duration_ms;
3343                    match verdict::extract_json::<FixReport>(&o.text) {
3344                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3345                            self.state.event(
3346                                "fix",
3347                                format!(
3348                                    "round {round}: fixer's adoption report recovered after \
3349                                     {attempts} continuation(s)"
3350                                ),
3351                            );
3352                            return (
3353                                seat,
3354                                Some(report),
3355                                None,
3356                                ContinuationRecord {
3357                                    attempts,
3358                                    cumulative_wait_ms,
3359                                    outcome: ContinuationOutcome::Resumed,
3360                                },
3361                            );
3362                        }
3363                        // The report parsed, but this same reply's own
3364                        // CommandEvidence — the identical record `state.jobs`
3365                        // renders — names a command whose CLI never
3366                        // confirmed an exit status. Read together, that is
3367                        // not a resolved answer: keep nudging rather than
3368                        // accept a report standing next to a command the
3369                        // seat's own CLI cannot vouch for.
3370                        Ok(_) => {
3371                            last_err = "the reply parsed, but it reported a command whose own CLI \
3372                                 never confirmed an exit status"
3373                                .to_owned();
3374                        }
3375                        Err(e) => last_err = e.to_string(),
3376                    }
3377                }
3378                AgentOutcome::Quota(o) => {
3379                    cumulative_wait_ms += o.duration_ms;
3380                    self.state.quota.push(QuotaLoss {
3381                        seat: seat.key.clone(),
3382                        node: "fix".to_owned(),
3383                        at: Timestamp::now(),
3384                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3385                    });
3386                    self.state.event(
3387                        "fix",
3388                        format!(
3389                            "round {round}: continuation rate limited (quota); not retrying now"
3390                        ),
3391                    );
3392                    return (
3393                        seat,
3394                        None,
3395                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3396                        ContinuationRecord {
3397                            attempts,
3398                            cumulative_wait_ms,
3399                            outcome: ContinuationOutcome::QuotaLost,
3400                        },
3401                    );
3402                }
3403                AgentOutcome::Dropped(o) => {
3404                    cumulative_wait_ms += o.duration_ms;
3405                    let why = o
3406                        .dropped
3407                        .as_ref()
3408                        .map(|d| d.why.as_str())
3409                        .unwrap_or("the CLI ended the stream without delivering its answer");
3410                    last_err = format!("the CLI dropped the stream ({why})");
3411                }
3412                AgentOutcome::Failed(e) => last_err = e,
3413                // The continuation was never started; the report stays lost
3414                // and the caller records that, without any handover.
3415                AgentOutcome::ResourceBlocked(e) => {
3416                    return (
3417                        seat,
3418                        None,
3419                        Some(format!("the continuation was not started: {e}")),
3420                        ContinuationRecord {
3421                            attempts: attempts.saturating_sub(1),
3422                            cumulative_wait_ms,
3423                            outcome: ContinuationOutcome::NoSession,
3424                        },
3425                    );
3426                }
3427            }
3428        }
3429    }
3430
3431    fn after_implement(&mut self) -> Result<()> {
3432        if !self.state.candidate_preservation.is_empty() {
3433            bail!("candidate preservation incomplete; judging is unsafe");
3434        }
3435        // Scan every candidate patch once the set is complete.
3436        if self.state.leaks.is_empty() {
3437            let cfg = self.state.config.blind.clone();
3438            let mut leaks = Vec::new();
3439            for c in &self.state.candidates {
3440                let Some(patch) =
3441                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3442                else {
3443                    continue;
3444                };
3445                leaks.extend(blind::scan(
3446                    &format!("candidate {} patch", c.label),
3447                    &patch,
3448                    &cfg.vendor_tokens,
3449                ));
3450            }
3451            if !leaks.is_empty() {
3452                let summary = leaks
3453                    .iter()
3454                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3455                    .collect::<Vec<_>>()
3456                    .join(", ");
3457                match cfg.on_leak {
3458                    LeakPolicy::Fail => {
3459                        self.state.status = RunStatus::Failed;
3460                        self.state
3461                            .event("blind", format!("vendor text in a patch: {summary}"));
3462                        self.state.leaks = leaks;
3463                        self.state.save()?;
3464                        self.settle_questions();
3465                        bail!(
3466                            "blind.on_leak = \"fail\" and vendor text reached a \
3467                             judged patch: {summary}"
3468                        );
3469                    }
3470                    LeakPolicy::Redact => self.state.event(
3471                        "blind",
3472                        format!("redacting vendor text for judging: {summary}"),
3473                    ),
3474                    LeakPolicy::Warn => self.state.event(
3475                        "blind",
3476                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3477                    ),
3478                }
3479                self.state.leaks = leaks;
3480            }
3481        }
3482
3483        if self.state.viable().is_empty() {
3484            if self.state.all_candidates_verified_noop() {
3485                // Every candidate agreed, with evidence the adoption guard
3486                // accepted, that nothing belongs in this worktree. That is
3487                // not the same fact as a candidate that simply failed to
3488                // write anything, and settling it as an ordinary `Failed`
3489                // (see `SCHEMA`'s doc for schema 10) is what let two of
3490                // task 391f's attempts burn a retry each re-discovering the
3491                // same already-landed fix. Terminal either way, so `judge`
3492                // must never run over an empty candidate set — unlike the
3493                // `Failed` branch below this returns `Ok`, not an error:
3494                // nothing here failed.
3495                self.state.status = RunStatus::VerifiedNoop;
3496                self.state.save()?;
3497                self.settle_questions();
3498                return Ok(());
3499            }
3500            self.state.status = RunStatus::Failed;
3501            self.state.save()?;
3502            self.settle_questions();
3503            bail!("no candidate produced a change; nothing to judge");
3504        }
3505        self.state.status = RunStatus::Judging;
3506        self.state.save()?;
3507        Ok(())
3508    }
3509
3510    // --------------------------------------------------------------- judge
3511
3512    async fn judge(&mut self) -> Result<()> {
3513        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3514        // agent files with `magi task add` name the run that paid for it. The
3515        // prompt overlay is cloned alongside it because the waves borrow it
3516        // while `self` is mutably borrowed by the node's own bookkeeping.
3517        let run_id = self.state.id.clone();
3518        let prompts = self.state.config.prompts.clone();
3519        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3520            return Ok(());
3521        }
3522        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3523        if viable.len() == 1 {
3524            // Recorded so this is a one-time event: `judgements` stays empty
3525            // either way, which without this flag is indistinguishable from
3526            // "not yet judged" on the next reentry — and status is left
3527            // untouched, so a later node's conclusion (e.g. `Blocked` after
3528            // the review budget ran out) survives a resume instead of being
3529            // clobbered back to `Judging` by this node running again.
3530            self.state.judge_skipped = true;
3531            self.state.event(
3532                "judge",
3533                format!(
3534                    "only candidate {} produced a change; judging skipped",
3535                    viable[0].label
3536                ),
3537            );
3538            self.state.save()?;
3539            return Ok(());
3540        }
3541        self.state.status = RunStatus::Judging;
3542
3543        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3544        let language = self.state.config.graph.language.clone();
3545        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3546        let sessions = self.state.config.graph.sessions;
3547        let artifacts = agent::artifacts_dir(&self.state.dir());
3548        let root = self.state.worktree_root();
3549        let base_short = short(&self.state.base_commit);
3550
3551        let mut jobs = Vec::new();
3552        let mut orders = Vec::new();
3553        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3554            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3555            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3556            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3557            let seat_key = format!("judge-{}", j + 1);
3558            let seat = self.seat(&seat_key, &spec.id);
3559            jobs.push(SeatJob {
3560                prompt: prompt::judge(
3561                    &self.state.instruction,
3562                    &views,
3563                    self.roles.judges.len(),
3564                    &base_short,
3565                    &language,
3566                ),
3567                spec,
3568                seat,
3569                cwd: root.join(format!("judge-{}", j + 1)),
3570                timeout,
3571                allow_write: false,
3572                sessions,
3573                artifacts: artifacts.clone(),
3574                stem: format!("judge-{}", j + 1),
3575                handover: None,
3576            });
3577        }
3578
3579        self.state.event(
3580            "judge",
3581            format!(
3582                "{} judges ranking {} candidates blind",
3583                jobs.len(),
3584                viable.len()
3585            ),
3586        );
3587        let labels_for_check = labels.clone();
3588        let mut quota_losses = Vec::new();
3589        let cache = self.state.config.cache_dir();
3590        let ctx = WaveCtx {
3591            carry_seats: false,
3592            pauses: Some((&self.pause, &self.interrupt)),
3593            run: &run_id,
3594            node: "judge",
3595            prompts: &prompts,
3596            cache: cache.as_deref(),
3597            round: None,
3598        };
3599        let results = ask_json_wave::<Ranking>(
3600            jobs,
3601            Arc::clone(&self.sem),
3602            self.state.config.graph.retries,
3603            &self.roles.judge_roster,
3604            &ctx,
3605            &mut quota_losses,
3606            &mut self.state,
3607            &move |r: &Ranking| r.validate(&labels_for_check),
3608        )
3609        .await;
3610        if self.state.resource_block.is_some() {
3611            return self.stop_for_resource();
3612        }
3613        self.state.quota.extend(quota_losses);
3614
3615        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3616            let agent_id = seat.agent.clone();
3617            self.state.seats.insert(seat.key.clone(), seat);
3618            let mut record = Judgement {
3619                judge: j + 1,
3620                seat: format!("judge-{}", j + 1),
3621                agent: agent_id,
3622                ranking: Vec::new(),
3623                reasons: BTreeMap::new(),
3624                confidence: None,
3625                strengths: BTreeMap::new(),
3626                order: orders[j].clone(),
3627                failed: None,
3628                duration_ms: 0,
3629            };
3630            match res {
3631                Ok((ranking, out)) => {
3632                    record.ranking = ranking.normalized();
3633                    record.reasons = ranking.reasons;
3634                    record.strengths = ranking.strengths;
3635                    record.confidence = ranking.confidence;
3636                    record.duration_ms = out.duration_ms;
3637                    self.state.event(
3638                        "judge",
3639                        format!(
3640                            "judge {} ranked {}",
3641                            j + 1,
3642                            record.ranking.iter().collect::<String>()
3643                        ),
3644                    );
3645                }
3646                Err(e) => {
3647                    record.failed = Some(e.to_string());
3648                    self.state
3649                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3650                }
3651            }
3652            self.state.judgements.push(record);
3653            self.state.save()?;
3654        }
3655        Ok(())
3656    }
3657
3658    // ---------------------------------------------------------- deliberate
3659
3660    async fn deliberate(&mut self) -> Result<()> {
3661        if self.hold_base_sync().await? {
3662            return Ok(());
3663        }
3664        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3665        // agent files with `magi task add` name the run that paid for it. The
3666        // prompt overlay is cloned alongside it because the waves borrow it
3667        // while `self` is mutably borrowed by the node's own bookkeeping.
3668        let run_id = self.state.id.clone();
3669        let prompts = self.state.config.prompts.clone();
3670        if !self.state.deliberation.is_empty() || self.state.tally.is_some() {
3671            return Ok(());
3672        }
3673        let tops: Vec<char> = self
3674            .state
3675            .judgements
3676            .iter()
3677            .filter_map(|j| j.ranking.first().copied())
3678            .collect();
3679        let rounds = self.state.config.graph.deliberate_rounds;
3680        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3681            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3682                self.state.event(
3683                    "deliberate",
3684                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3685                );
3686            }
3687            self.state.status = RunStatus::Voting;
3688            self.state.save()?;
3689            return Ok(());
3690        }
3691
3692        self.state.status = RunStatus::Deliberating;
3693        self.state.event(
3694            "deliberate",
3695            format!(
3696                "split: first choices were {} — opening {rounds} round(s)",
3697                tops.iter().collect::<String>()
3698            ),
3699        );
3700
3701        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3702        let language = self.state.config.graph.language.clone();
3703        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3704        let sessions = self.state.config.graph.sessions;
3705        let artifacts = agent::artifacts_dir(&self.state.dir());
3706        let root = self.state.worktree_root();
3707        let base_short = short(&self.state.base_commit);
3708
3709        // Judges argue in sequence so that a turn can answer the one before it;
3710        // that is the difference between deliberation and three parallel
3711        // monologues.
3712        for round in 1..=rounds {
3713            let mut turns: Vec<DeliberationTurn> = Vec::new();
3714            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3715                if self.state.judgements[j].failed.is_some() {
3716                    continue;
3717                }
3718                let seat_key = format!("judge-{}", j + 1);
3719                let spec = self.occupant(&seat_key, spec);
3720                let mut seat = self.seat(&seat_key, &spec.id);
3721                let transcript = self.transcript(&turns, j);
3722                let build = |context: Option<&str>| {
3723                    prompt::deliberate(
3724                        &self.state.instruction,
3725                        context,
3726                        &transcript,
3727                        round,
3728                        rounds,
3729                        &language,
3730                    )
3731                };
3732                let block = self.candidate_block(&viable, &base_short);
3733                let full = build(Some(&block));
3734                let text = if has_context(&spec, &seat, sessions) {
3735                    build(None)
3736                } else {
3737                    full.clone()
3738                };
3739                let job = SeatJob {
3740                    spec,
3741                    seat: seat.clone(),
3742                    prompt: text,
3743                    cwd: root.join(format!("judge-{}", j + 1)),
3744                    timeout,
3745                    allow_write: false,
3746                    sessions,
3747                    artifacts: artifacts.clone(),
3748                    stem: format!("delib-{round}-judge-{}", j + 1),
3749                    handover: Some(full),
3750                };
3751                let cache = self.state.config.cache_dir();
3752                let ctx = WaveCtx {
3753                    carry_seats: false,
3754                    pauses: Some((&self.pause, &self.interrupt)),
3755                    run: &run_id,
3756                    node: "deliberate",
3757                    prompts: &prompts,
3758                    cache: cache.as_deref(),
3759                    round: None,
3760                };
3761                // A turn is never nudged (`retries` 0): a failed seat is
3762                // handed to the next roster agent, which gets the full
3763                // context. An empty answer is a turn, not a failure.
3764                let mut losses = Vec::new();
3765                let mut results = ask_wave_with::<String>(
3766                    vec![job],
3767                    Arc::clone(&self.sem),
3768                    0,
3769                    &self.roles.judge_roster,
3770                    &ctx,
3771                    &mut losses,
3772                    &mut self.state,
3773                    &|text: &str| {
3774                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3775                    },
3776                )
3777                .await;
3778                self.state.quota.extend(losses);
3779                let (updated, res, _) = results.pop().expect("one job in, one result out");
3780                seat = updated;
3781                let agent_id = seat.agent.clone();
3782                self.state.seats.insert(seat.key.clone(), seat);
3783                let body = match res {
3784                    Ok((body, _)) => body,
3785                    // Skip the seat; a CLI's raw error JSON is never read as
3786                    // this judge's position.
3787                    Err(e) => {
3788                        self.state
3789                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3790                        continue;
3791                    }
3792                };
3793                let tentative = verdict::extract_json::<Position>(&body)
3794                    .ok()
3795                    .and_then(|p| p.tentative)
3796                    .and_then(|s| s.trim().chars().next())
3797                    .map(|c| c.to_ascii_uppercase());
3798                self.state.event(
3799                    "deliberate",
3800                    format!(
3801                        "round {round}: judge {} now favours {}",
3802                        j + 1,
3803                        tentative.map_or("—".to_owned(), |c| c.to_string())
3804                    ),
3805                );
3806                turns.push(DeliberationTurn {
3807                    judge: j + 1,
3808                    agent: agent_id,
3809                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3810                    tentative,
3811                });
3812            }
3813            self.state
3814                .deliberation
3815                .push(DeliberationRound { round, turns });
3816            self.state.save()?;
3817        }
3818
3819        self.state.status = RunStatus::Voting;
3820        self.state.save()?;
3821        Ok(())
3822    }
3823
3824    // ---------------------------------------------------------------- vote
3825
3826    async fn vote(&mut self) -> Result<()> {
3827        if self.hold_base_sync().await? {
3828            return Ok(());
3829        }
3830        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3831        // agent files with `magi task add` name the run that paid for it. The
3832        // prompt overlay is cloned alongside it because the waves borrow it
3833        // while `self` is mutably borrowed by the node's own bookkeeping.
3834        let run_id = self.state.id.clone();
3835        let prompts = self.state.config.prompts.clone();
3836        if !self.state.votes.is_empty() || self.state.tally.is_some() {
3837            return Ok(());
3838        }
3839        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3840        if viable.len() == 1 {
3841            return Ok(());
3842        }
3843        self.state.status = RunStatus::Voting;
3844
3845        let language = self.state.config.graph.language.clone();
3846        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3847        let sessions = self.state.config.graph.sessions;
3848        let artifacts = agent::artifacts_dir(&self.state.dir());
3849        let root = self.state.worktree_root();
3850        let base_short = short(&self.state.base_commit);
3851        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3852
3853        let mut jobs = Vec::new();
3854        let mut seats_at = Vec::new();
3855        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3856            if self
3857                .state
3858                .judgements
3859                .get(j)
3860                .is_some_and(|r| r.failed.is_some())
3861            {
3862                continue;
3863            }
3864            let seat_key = format!("judge-{}", j + 1);
3865            let spec = self.occupant(&seat_key, spec);
3866            let seat = self.seat(&seat_key, &spec.id);
3867            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3868            let text = if has_context(&spec, &seat, sessions) {
3869                prompt::final_vote(&viable, &language)
3870            } else {
3871                full.clone()
3872            };
3873            jobs.push(SeatJob {
3874                spec,
3875                seat,
3876                prompt: text,
3877                cwd: root.join(format!("judge-{}", j + 1)),
3878                timeout,
3879                allow_write: false,
3880                sessions,
3881                artifacts: artifacts.clone(),
3882                stem: format!("vote-judge-{}", j + 1),
3883                handover: Some(full),
3884            });
3885            seats_at.push(j);
3886        }
3887
3888        self.state.event(
3889            "vote",
3890            format!(
3891                "collecting {} final votes one by one, privately",
3892                jobs.len()
3893            ),
3894        );
3895        let allowed = viable.clone();
3896        let mut quota_losses = Vec::new();
3897        let cache = self.state.config.cache_dir();
3898        let ctx = WaveCtx {
3899            carry_seats: false,
3900            pauses: Some((&self.pause, &self.interrupt)),
3901            run: &run_id,
3902            node: "vote",
3903            prompts: &prompts,
3904            cache: cache.as_deref(),
3905            round: None,
3906        };
3907        let results = ask_json_wave::<FinalVote>(
3908            jobs,
3909            Arc::clone(&self.sem),
3910            self.state.config.graph.retries,
3911            &self.roles.judge_roster,
3912            &ctx,
3913            &mut quota_losses,
3914            &mut self.state,
3915            &move |v: &FinalVote| match v.label() {
3916                Some(c) if allowed.contains(&c) => Ok(()),
3917                other => bail!("vote {other:?} is not one of {allowed:?}"),
3918            },
3919        )
3920        .await;
3921        if self.state.resource_block.is_some() {
3922            return self.stop_for_resource();
3923        }
3924        self.state.quota.extend(quota_losses);
3925
3926        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3927            let agent_id = seat.agent.clone();
3928            self.state.seats.insert(seat.key.clone(), seat);
3929            let initial = self
3930                .state
3931                .judgements
3932                .get(j)
3933                .and_then(|r| r.ranking.first().copied());
3934            let mut record = VoteRecord {
3935                judge: j + 1,
3936                agent: agent_id,
3937                vote: None,
3938                reason: String::new(),
3939                changed: false,
3940            };
3941            match res {
3942                Ok((v, _)) => {
3943                    record.vote = v.label();
3944                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3945                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3946                    self.state.event(
3947                        "vote",
3948                        format!(
3949                            "judge {} voted {}{}",
3950                            j + 1,
3951                            record.vote.unwrap_or('?'),
3952                            if record.changed { " (changed)" } else { "" }
3953                        ),
3954                    );
3955                }
3956                Err(e) => {
3957                    self.state
3958                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3959                }
3960            }
3961            self.state.votes.push(record);
3962            self.state.save()?;
3963        }
3964        Ok(())
3965    }
3966
3967    // --------------------------------------------------------------- tally
3968
3969    fn tally(&mut self) -> Result<()> {
3970        if self.state.tally.is_some() {
3971            return Ok(());
3972        }
3973        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3974        let tops: Vec<char> = self
3975            .state
3976            .judgements
3977            .iter()
3978            .filter_map(|j| j.ranking.first().copied())
3979            .collect();
3980        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3981
3982        // A judge whose private vote failed still counted once, in the initial
3983        // ranking; using it beats discarding a whole seat.
3984        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3985        let mut cast: Vec<char> = Vec::new();
3986        for (i, j) in self.state.judgements.iter().enumerate() {
3987            let vote = self
3988                .state
3989                .votes
3990                .iter()
3991                .find(|v| v.judge == i + 1)
3992                .and_then(|v| v.vote)
3993                .or_else(|| j.ranking.first().copied());
3994            if let Some(v) = vote {
3995                *first_choice.entry(v).or_insert(0) += 1;
3996                cast.push(v);
3997            }
3998        }
3999
4000        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4001        for j in &self.state.judgements {
4002            let n = j.ranking.len();
4003            for (pos, label) in j.ranking.iter().enumerate() {
4004                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4005            }
4006        }
4007
4008        let best = first_choice.values().copied().max().unwrap_or(0);
4009        let mut leaders: Vec<char> = first_choice
4010            .iter()
4011            .filter(|(_, v)| **v == best)
4012            .map(|(k, _)| *k)
4013            .collect();
4014        let mut tie_break = None;
4015        if leaders.len() > 1 {
4016            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4017            let borda_leaders: Vec<char> = leaders
4018                .iter()
4019                .copied()
4020                .filter(|l| borda[l] == top_borda)
4021                .collect();
4022            tie_break = Some(if borda_leaders.len() == 1 {
4023                format!(
4024                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4025                    leaders.len()
4026                )
4027            } else {
4028                format!(
4029                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4030                    leaders.len()
4031                )
4032            });
4033            leaders = borda_leaders;
4034            leaders.sort_unstable();
4035        }
4036        let winner = *leaders
4037            .first()
4038            .or(viable.first())
4039            .context("no candidate to declare a winner from")?;
4040
4041        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4042        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4043        let deliberated = !self.state.deliberation.is_empty();
4044
4045        // Whose verdict is this? A rate-limited seat is absent even if it
4046        // ranked before the limit hit, so presence is measured against the
4047        // recorded losses, not just "did a ranking ever appear".
4048        let quota_seats: std::collections::BTreeSet<&str> =
4049            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4050        let mut present = 0usize;
4051        for (i, j) in self.state.judgements.iter().enumerate() {
4052            if quota_seats.contains(j.seat.as_str()) {
4053                continue;
4054            }
4055            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4056            let voted = self
4057                .state
4058                .votes
4059                .iter()
4060                .any(|v| v.judge == i + 1 && v.vote.is_some());
4061            if ranked || voted {
4062                present += 1;
4063            }
4064        }
4065        // Strict majority of the configured panel. A bare majority is real
4066        // signal we can act on, while a minority verdict must never stand in
4067        // for a healthy one. A one-candidate run needs no panel at all, and
4068        // `judges` stays `0` rather than the roster size a panel that never
4069        // sat would otherwise be credited with.
4070        let needs_quorum = viable.len() > 1;
4071        let judges_total = if needs_quorum {
4072            self.roles.judges.len()
4073        } else {
4074            0
4075        };
4076        let quorum = if needs_quorum {
4077            judges_total / 2 + 1
4078        } else {
4079            0
4080        };
4081        let met_quorum = !needs_quorum || present >= quorum;
4082        let uncontested = (!needs_quorum).then(|| {
4083            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4084        });
4085
4086        self.state.event(
4087            "tally",
4088            match &uncontested {
4089                Some(reason) => format!("winner {winner} — {reason}"),
4090                None => format!(
4091                    "winner {winner} — votes {} | initial {} | {} changed | \
4092                     {present}/{judges_total} judges{}",
4093                    first_choice
4094                        .iter()
4095                        .map(|(k, v)| format!("{k}:{v}"))
4096                        .collect::<Vec<_>>()
4097                        .join(" "),
4098                    if unanimous_initial {
4099                        "unanimous"
4100                    } else {
4101                        "split"
4102                    },
4103                    changed_votes,
4104                    if met_quorum {
4105                        String::new()
4106                    } else {
4107                        format!(" — below quorum ({quorum} required)")
4108                    },
4109                ),
4110            },
4111        );
4112        if !met_quorum {
4113            self.state.event(
4114                "stall",
4115                format!(
4116                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4117                     the run stops here, resumable"
4118                ),
4119            );
4120        }
4121        self.state.tally = Some(Tally {
4122            first_choice,
4123            borda,
4124            winner,
4125            rankings: tops.len(),
4126            unanimous_initial,
4127            deliberated,
4128            changed_votes,
4129            unanimous_final,
4130            tie_break,
4131            judges: judges_total,
4132            present,
4133            quorum,
4134            met_quorum,
4135            uncontested,
4136        });
4137        self.state.status = if met_quorum {
4138            RunStatus::Reviewing
4139        } else {
4140            RunStatus::Stalled
4141        };
4142        self.state.save()?;
4143        Ok(())
4144    }
4145
4146    // ------------------------------------------------------------- recover
4147
4148    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4149    /// resumed toward completion once the transient cause clears.
4150    ///
4151    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4152    /// it toward the quorum, which is exactly the set of seats whose absence
4153    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4154    /// not depend on which node happened to hit the limit), or an ordinary
4155    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4156    /// seat is never disturbed.
4157    ///
4158    /// A seat that now answers with a usable ranking is "recovered": its
4159    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4160    /// `tally` counts it present again), and its vote re-collected. A seat that
4161    /// still fails keeps its loss and stays absent.
4162    ///
4163    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4164    /// to review/gate/merge), `false` when it is still below quorum (the run
4165    /// stays `Stalled`, still resumable for a later retry).
4166    #[allow(clippy::too_many_lines)]
4167    async fn recover_stall(&mut self) -> Result<bool> {
4168        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4169        // agent files with `magi task add` name the run that paid for it. The
4170        // prompt overlay is cloned alongside it because the waves borrow it
4171        // while `self` is mutably borrowed by the node's own bookkeeping.
4172        let run_id = self.state.id.clone();
4173        let prompts = self.state.config.prompts.clone();
4174        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4175        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4176        // is what keeps a non-quota collapse — or a quota loss recorded at the
4177        // deliberate node — from being a permanent dead-end on `--resume`.
4178        let quota_seats: BTreeSet<&str> =
4179            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4180        let absent: Vec<String> = self
4181            .state
4182            .judgements
4183            .iter()
4184            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4185            .map(|j| j.seat.clone())
4186            .collect();
4187        if absent.is_empty() {
4188            return Ok(false);
4189        }
4190        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4191        if viable.len() <= 1 {
4192            return Ok(false);
4193        }
4194        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4195        let language = self.state.config.graph.language.clone();
4196        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4197        let sessions = self.state.config.graph.sessions;
4198        let artifacts = agent::artifacts_dir(&self.state.dir());
4199        let root = self.state.worktree_root();
4200        let base_short = short(&self.state.base_commit);
4201        let candidates: Vec<Candidate> = viable.clone();
4202
4203        // Map each absent seat key to its 0-based position in `roles.judges`.
4204        let mut positions: Vec<usize> = absent
4205            .iter()
4206            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4207            .collect();
4208        if positions.is_empty() {
4209            return Ok(false);
4210        }
4211        positions.sort_unstable();
4212        positions.dedup();
4213
4214        // Re-rank the lost seats, one blind prompt each.
4215        let mut judge_jobs = Vec::new();
4216        for &j in &positions {
4217            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4218            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4219            let seat_key = format!("judge-{}", j + 1);
4220            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4221            let seat = self.seat(&seat_key, &spec.id);
4222            judge_jobs.push(SeatJob {
4223                spec,
4224                seat,
4225                prompt: prompt::judge(
4226                    &self.state.instruction,
4227                    &views,
4228                    self.roles.judges.len(),
4229                    &base_short,
4230                    &language,
4231                ),
4232                cwd: root.join(seat_key),
4233                timeout,
4234                allow_write: false,
4235                sessions,
4236                artifacts: artifacts.clone(),
4237                stem: format!("judge-{}-recover", j + 1),
4238                handover: None,
4239            });
4240        }
4241
4242        let labels_for_check = labels.clone();
4243        let mut judge_losses = Vec::new();
4244        let retries = self.state.config.graph.retries;
4245        let cache = self.state.config.cache_dir();
4246        let ctx = WaveCtx {
4247            carry_seats: false,
4248            pauses: Some((&self.pause, &self.interrupt)),
4249            run: &run_id,
4250            node: "judge",
4251            prompts: &prompts,
4252            cache: cache.as_deref(),
4253            round: None,
4254        };
4255        let results = ask_json_wave::<Ranking>(
4256            judge_jobs,
4257            Arc::clone(&self.sem),
4258            retries,
4259            &self.roles.judge_roster,
4260            &ctx,
4261            &mut judge_losses,
4262            &mut self.state,
4263            &move |r: &Ranking| r.validate(&labels_for_check),
4264        )
4265        .await;
4266        if self.state.resource_block.is_some() {
4267            self.stop_for_resource()?;
4268            return Ok(false);
4269        }
4270
4271        // Refresh the judgement of every seat that ranked again.
4272        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4273        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4274            let agent_id = seat.agent.clone();
4275            self.state.seats.insert(seat.key.clone(), seat);
4276            let record = &mut self.state.judgements[j];
4277            match res {
4278                Ok((ranking, out)) => {
4279                    record.agent = agent_id;
4280                    record.ranking = ranking.normalized();
4281                    record.reasons = ranking.reasons;
4282                    record.strengths = ranking.strengths;
4283                    record.confidence = ranking.confidence;
4284                    record.failed = None;
4285                    record.duration_ms = out.duration_ms;
4286                    recovered.insert(j);
4287                    self.state.event(
4288                        "recover",
4289                        format!("judge {} ranked again after the limit", j + 1),
4290                    );
4291                }
4292                Err(e) => {
4293                    self.state
4294                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4295                }
4296            }
4297        }
4298
4299        // Re-ask the votes of the seats that recovered a ranking.
4300        let mut vote_jobs = Vec::new();
4301        let mut vote_pos: Vec<usize> = Vec::new();
4302        for &j in &recovered {
4303            let seat_key = format!("judge-{}", j + 1);
4304            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4305            let seat = self.seat(&seat_key, &spec.id);
4306            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4307            let text = if has_context(&spec, &seat, sessions) {
4308                prompt::final_vote(&labels, &language)
4309            } else {
4310                full.clone()
4311            };
4312            vote_jobs.push(SeatJob {
4313                spec,
4314                seat,
4315                prompt: text,
4316                cwd: root.join(seat_key),
4317                timeout,
4318                allow_write: false,
4319                sessions,
4320                artifacts: artifacts.clone(),
4321                stem: format!("vote-judge-{}-recover", j + 1),
4322                handover: Some(full),
4323            });
4324            vote_pos.push(j);
4325        }
4326        let allowed = labels.clone();
4327        let mut vote_losses = Vec::new();
4328        let vote_retries = self.state.config.graph.retries;
4329        let vote_cache = self.state.config.cache_dir();
4330        let ctx = WaveCtx {
4331            carry_seats: false,
4332            pauses: Some((&self.pause, &self.interrupt)),
4333            run: &run_id,
4334            node: "vote",
4335            prompts: &prompts,
4336            cache: vote_cache.as_deref(),
4337            round: None,
4338        };
4339        let votes = ask_json_wave::<FinalVote>(
4340            vote_jobs,
4341            Arc::clone(&self.sem),
4342            vote_retries,
4343            &self.roles.judge_roster,
4344            &ctx,
4345            &mut vote_losses,
4346            &mut self.state,
4347            &move |v: &FinalVote| match v.label() {
4348                Some(c) if allowed.contains(&c) => Ok(()),
4349                other => bail!("vote {other:?} is not one of {allowed:?}"),
4350            },
4351        )
4352        .await;
4353        if self.state.resource_block.is_some() {
4354            self.stop_for_resource()?;
4355            return Ok(false);
4356        }
4357        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4358            let agent_id = seat.agent.clone();
4359            self.state.seats.insert(seat.key.clone(), seat);
4360            match res {
4361                Ok((v, _)) => {
4362                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4363                        rec.vote = v.label();
4364                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4365                    } else {
4366                        self.state.votes.push(VoteRecord {
4367                            judge: j + 1,
4368                            agent: agent_id,
4369                            vote: v.label(),
4370                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4371                            changed: false,
4372                        });
4373                    }
4374                    self.state.event(
4375                        "recover",
4376                        format!("judge {} voted again after the limit", j + 1),
4377                    );
4378                }
4379                Err(e) => {
4380                    self.state
4381                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4382                }
4383            }
4384        }
4385
4386        // A seat that ranked again is present even if its re-vote failed —
4387        // `tally` falls back to the initial ranking's first choice — so clear
4388        // its quota loss. Seats that still fail keep theirs and stay absent.
4389        let recovered_keys: BTreeSet<String> = recovered
4390            .iter()
4391            .map(|&j| format!("judge-{}", j + 1))
4392            .collect();
4393        self.state
4394            .quota
4395            .retain(|q| !recovered_keys.contains(&q.seat));
4396        // A seat that hit the limit again is a fresh loss, not the old one:
4397        // replace the stale entry so the history stays one-per-seat and the
4398        // daemon can tell this attempt's loss from a previous session's.
4399        for loss in judge_losses.into_iter().chain(vote_losses) {
4400            if recovered_keys.contains(&loss.seat) {
4401                continue;
4402            }
4403            self.state.quota.retain(|q| q.seat != loss.seat);
4404            self.state.quota.push(loss);
4405        }
4406
4407        // Recompute the verdict from the refreshed panel.
4408        self.state.tally = None;
4409        self.tally()?;
4410        Ok(self
4411            .state
4412            .tally
4413            .as_ref()
4414            .map(|t| t.met_quorum)
4415            .unwrap_or(false))
4416    }
4417
4418    // ---------------------------------------------------------------- graft
4419
4420    /// Port the strengths the judges attributed to the losing candidates onto
4421    /// the winner, once, between the tally and the review loop.
4422    ///
4423    /// Best-effort by construction: it returns nothing and fails nothing. A
4424    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4425    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4426    /// the run carries on as if the phase were off. See [`graft`] for the
4427    /// decisions and `AGENTS.md` for the invariants.
4428    ///
4429    /// The one exception is a rollback that could not be confirmed: the
4430    /// rejected commit may still be the winner's tip, so the run is blocked
4431    /// (resumable; the record stays `Running`, which makes a resume retry the
4432    /// restore) instead of reviewing, gating and merging it.
4433    async fn graft(&mut self) -> Result<()> {
4434        let Err(e) = self.graft_inner().await else {
4435            return Ok(());
4436        };
4437        if e.is::<crate::repo_lock::ResourceBlocked>() {
4438            return Err(e);
4439        }
4440        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4441            self.state.status = RunStatus::Blocked;
4442            self.state.event("graft", format!("blocked: {e:#}"));
4443            self.state.save()?;
4444            return Err(e);
4445        }
4446        // An error after the `Running` record was saved may have left the
4447        // fixer's work in the tree: the record must not stay unsettled
4448        // (a later resume would reset past the review fixes), so confirm the
4449        // restore now, or block.
4450        if let Some(rec) = self.state.graft.clone()
4451            && !rec.outcome.settled()
4452        {
4453            if let Some(winner) = self.state.winner().cloned() {
4454                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4455                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4456                        return Err(re);
4457                    }
4458                    self.state.status = RunStatus::Blocked;
4459                    self.state.event("graft", format!("blocked: {re:#}"));
4460                    self.state.save()?;
4461                    return Err(re);
4462                }
4463            }
4464            if let Some(g) = self.state.graft.as_mut() {
4465                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4466            }
4467        }
4468        self.state.event("graft", format!("skipped: {e:#}"));
4469        if let Err(e) = self.state.save() {
4470            tracing::warn!("save after graft: {e:#}");
4471        }
4472        Ok(())
4473    }
4474
4475    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4476    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4477    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4478        let _coordination = self.coordinate(wt).await?;
4479        crate::recovery::restore_repair(wt, pre)
4480            .await
4481            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4482    }
4483
4484    /// A skip is a run event, recorded once however often the node is re-entered.
4485    fn graft_skipped(&mut self, why: &str) {
4486        let message = format!("skipped: {why}");
4487        if !self
4488            .state
4489            .events
4490            .iter()
4491            .any(|e| e.node == "graft" && e.message == message)
4492        {
4493            self.state.event("graft", message);
4494        }
4495    }
4496
4497    async fn graft_inner(&mut self) -> Result<()> {
4498        // A round that was started and never finished (crash, kill): the
4499        // fixer may have left anything in the tree, so put the winner back.
4500        if let Some(rec) = self.state.graft.clone()
4501            && !rec.outcome.settled()
4502        {
4503            // Reviews exist only if the graft ended without its record being
4504            // settled (a swallowed error): the tree has moved on since, and
4505            // a reset to `pre` would discard the review fixes.
4506            if self.state.reviews.is_empty()
4507                && let Some(winner) = self.state.winner().cloned()
4508            {
4509                self.graft_restore(&winner.worktree, &rec.pre).await?;
4510            }
4511            if rec.outcome == GraftOutcome::WaitingForRestore {
4512                self.state.graft = None;
4513                self.state.save()?;
4514            } else {
4515                if let Some(g) = self.state.graft.as_mut() {
4516                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4517                }
4518                self.state.event(
4519                    "graft",
4520                    "rolled back: the previous round was interrupted; the winner is untouched",
4521                );
4522                self.state.save()?;
4523                return Ok(());
4524            }
4525        }
4526        let plan = match graft::plan(&self.state) {
4527            Ok(plan) => plan,
4528            Err(skip) => {
4529                if skip != graft::Skip::AlreadyDone {
4530                    self.graft_skipped(skip.reason());
4531                    self.state.save()?;
4532                }
4533                return Ok(());
4534            }
4535        };
4536        let Some(winner) = self.state.winner().cloned() else {
4537            return Ok(());
4538        };
4539        let wt = winner.worktree.clone();
4540        if !git::is_clean(&wt).await? {
4541            self.graft_skipped("the winner's worktree has uncommitted changes");
4542            self.state.save()?;
4543            return Ok(());
4544        }
4545        let pre = git::rev_parse(&wt, "HEAD").await?;
4546        // Saved before the fixer is called: a stop from here on is repaired
4547        // by the branch above on the next entry.
4548        self.state.graft = Some(GraftRecord {
4549            pre: pre.clone(),
4550            post: None,
4551            lines: None,
4552            limit: None,
4553            agent: None,
4554            outcome: GraftOutcome::Running,
4555            requested: graft::requested(&plan),
4556            adoption: None,
4557            adoption_error: None,
4558        });
4559        self.state.event(
4560            "graft",
4561            format!(
4562                "porting strengths of candidate(s) {} onto {}",
4563                plan.strengths
4564                    .iter()
4565                    .map(|(l, _, _)| l.to_string())
4566                    .collect::<Vec<_>>()
4567                    .join(", "),
4568                plan.winner
4569            ),
4570        );
4571        self.state.save()?;
4572
4573        let fix_prompt = prompt::graft(
4574            &self.state.instruction,
4575            plan.winner,
4576            &plan.strengths,
4577            &self.state.config.graph.language,
4578        );
4579        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4580        let sessions = self.state.config.graph.sessions;
4581        let artifacts = agent::artifacts_dir(&self.state.dir());
4582        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4583        let (_, seat, out) = self
4584            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4585                SeatJob {
4586                    prompt: fix_prompt.clone(),
4587                    spec,
4588                    seat,
4589                    cwd: wt.clone(),
4590                    timeout,
4591                    allow_write: true,
4592                    sessions,
4593                    artifacts: artifacts.clone(),
4594                    stem: "graft".to_owned(),
4595                    handover: None,
4596                }
4597            })
4598            .await;
4599        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4600            // Preserve the need to restore earlier fallback edits without
4601            // confusing an unstarted seat with an interrupted invocation.
4602            self.state.graft.as_mut().expect("graft record").outcome =
4603                GraftOutcome::WaitingForRestore;
4604            return self.stop_for_resource();
4605        }
4606        let mut why: Option<String> = None;
4607        match &out {
4608            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4609                why = Some("the fixer produced nothing usable".to_owned());
4610            }
4611            AgentOutcome::Ok(_) => {}
4612            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4613            AgentOutcome::Quota(o) => {
4614                self.state.quota.push(QuotaLoss {
4615                    seat: seat.key.clone(),
4616                    node: "graft".to_owned(),
4617                    at: Timestamp::now(),
4618                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4619                });
4620                why = Some("rate limited (quota)".to_owned());
4621            }
4622            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4623                why = Some(format!(
4624                    "the fixer failed: {}",
4625                    e.lines()
4626                        .next()
4627                        .unwrap_or("")
4628                        .chars()
4629                        .take(160)
4630                        .collect::<String>()
4631                ));
4632            }
4633        }
4634        let agent_id = seat.agent.clone();
4635        self.state.seats.insert(seat.key.clone(), seat);
4636
4637        let mut post = None;
4638        let mut lines = None;
4639        let _coordination = self.coordinate(&wt).await?;
4640        if why.is_none() {
4641            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4642            if let Ok(r) = &rescued {
4643                self.state.note_withheld("graft", &r.withheld);
4644            }
4645            let head = git::rev_parse(&wt, "HEAD").await?;
4646            // What is measured must be what gets reviewed and gated: a failed
4647            // rescue, or anything still uncommitted (withheld lockfiles
4648            // aside, which are never part of the tree), rejects the graft.
4649            let dirty = match &rescued {
4650                Ok(r) => git::status_porcelain(&wt)
4651                    .await
4652                    .map(|st| {
4653                        st.lines().any(|l| {
4654                            !r.withheld
4655                                .iter()
4656                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4657                        })
4658                    })
4659                    .unwrap_or(true),
4660                Err(_) => true,
4661            };
4662            if dirty {
4663                why = Some("the fixer left changes that could not be committed".to_owned());
4664            } else if head == pre {
4665                why = Some("the fixer changed nothing".to_owned());
4666            } else {
4667                lines = git::changed_lines(&wt, &pre, &head).await;
4668                post = Some(head);
4669            }
4670        }
4671        let (adoption, adoption_error) = match &out {
4672            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4673            _ => (None, None),
4674        };
4675        let requested = graft::requested(&plan);
4676        let outcome = match why {
4677            None => {
4678                self.state.event(
4679                    "graft",
4680                    format!(
4681                        "applied: {}",
4682                        lines.map_or_else(
4683                            || "changed lines unavailable".to_owned(),
4684                            |n| format!("{n} changed lines")
4685                        )
4686                    ),
4687                );
4688                GraftOutcome::Applied
4689            }
4690            Some(why) => {
4691                // Not settled until the restore is confirmed: the record
4692                // stays `Running`, so a resume retries it.
4693                crate::recovery::restore_repair(&wt, &pre)
4694                    .await
4695                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4696                self.state.event(
4697                    "graft",
4698                    format!("rolled back: {why}; the winner is untouched"),
4699                );
4700                GraftOutcome::RolledBack(why)
4701            }
4702        };
4703        self.state.graft = Some(GraftRecord {
4704            pre,
4705            post,
4706            lines,
4707            limit: None,
4708            agent: Some(agent_id),
4709            outcome,
4710            requested,
4711            adoption,
4712            adoption_error,
4713        });
4714        self.state.save()?;
4715        Ok(())
4716    }
4717
4718    // ----------------------------------------------------------------- fold
4719
4720    async fn fold_losers(&mut self) -> Result<()> {
4721        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4722            return Ok(());
4723        };
4724        if !self.state.candidate_preservation.is_empty() {
4725            bail!("candidate preservation incomplete; refusing cleanup");
4726        }
4727        let repo = self.state.repo.clone();
4728        let _coordination = self.coordinate(&repo).await?;
4729        let mut folded = Vec::new();
4730        for i in 0..self.state.candidates.len() {
4731            let c = &self.state.candidates[i];
4732            if c.label == winner || c.folded {
4733                continue;
4734            }
4735            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4736            remove_preserved_worktree(&repo, &wt).await?;
4737            if git::branch_exists(&repo, &branch).await?
4738                && !git::branch_delete(&repo, &branch).await?
4739            {
4740                bail!("candidate branch removal refused");
4741            }
4742            self.state.candidates[i].folded = true;
4743            self.state.save()?;
4744            folded.push(label.to_string());
4745        }
4746        // The judges are finished; their checkouts are pure cost from here.
4747        let root = self.state.worktree_root();
4748        for j in 1..=self.roles.judges.len() {
4749            let wt = root.join(format!("judge-{j}"));
4750            if wt.exists() {
4751                remove_preserved_worktree(&repo, &wt).await?;
4752            }
4753        }
4754        // The design-deliberation stage is finished by the time a tally
4755        // exists — same reasoning as the judges above.
4756        if self.state.config.graph.advise {
4757            for k in 1..=self.state.config.graph.advisors {
4758                let wt = root.join(format!("advisor-{k}"));
4759                if wt.exists() {
4760                    remove_preserved_worktree(&repo, &wt).await?;
4761                }
4762            }
4763        }
4764        if !folded.is_empty() {
4765            self.state
4766                .event("fold", format!("folded candidates {}", folded.join(", ")));
4767            self.state.save()?;
4768        }
4769        Ok(())
4770    }
4771
4772    // ------------------------------------------------------------ base sync
4773
4774    async fn hold_base_sync(&mut self) -> Result<bool> {
4775        self.hold_base_sync_as(None).await
4776    }
4777
4778    async fn hold_base_sync_kind(
4779        &mut self,
4780        kind: crate::base_sync_recovery::StopKind,
4781    ) -> Result<()> {
4782        self.hold_base_sync_as(Some(kind)).await?;
4783        Ok(())
4784    }
4785
4786    async fn hold_base_sync_as(
4787        &mut self,
4788        kind: Option<crate::base_sync_recovery::StopKind>,
4789    ) -> Result<bool> {
4790        if !self.state.base_sync_blocked() {
4791            self.state.base_sync_recovery.stop = None;
4792            return Ok(false);
4793        }
4794        let head = match self.state.winner() {
4795            Some(w) => git::rev_parse(&w.worktree, "HEAD")
4796                .await
4797                .unwrap_or_default(),
4798            None => String::new(),
4799        };
4800        self.state.record_base_sync_stop(&head);
4801        if let Some(kind) = kind {
4802            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
4803        }
4804        self.state.save()?;
4805        Ok(true)
4806    }
4807
4808    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4809    /// anything verifies it.
4810    ///
4811    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4812    /// read whatever is checked out in the winner's worktree. Left alone that
4813    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4814    /// it when the run *branched* - and a run takes long enough that the base
4815    /// has usually moved by the time it gets here. A gate that ran there
4816    /// answers "green on the commit this run started from", not "green on
4817    /// what is about to land", and the difference showed up three times in
4818    /// one day as a green run whose merge would have reverted a file another
4819    /// pull request had already landed.
4820    ///
4821    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4822    /// `land::Step::Rebase` calls, rather than a second implementation of the
4823    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4824    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4825    /// doc warns about ("two answers to a question nobody notices until a
4826    /// diff is wrong").
4827    ///
4828    /// Foreign-checkout reviews instead merge the base into the owned shadow
4829    /// worktree, preserving the ancestry required for fast-forward delivery.
4830    /// A conflicting merge is aborted and held for operator resolution.
4831    ///
4832    /// A conflict is not the end of the road: the standing rebase is handed
4833    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4834    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4835    /// land). Once it finishes, review and the gate run as usual on the
4836    /// rebased tree, which is where a breakage the new base caused is caught
4837    /// by the ordinary gate-fix round. magi resolves nothing itself.
4838    ///
4839    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4840    /// `state.base_sync.attempts`, is how many times the base is *rebased
4841    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4842    /// *conflict* was given to a fixer. When the fixer cannot finish the
4843    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4844    /// what was tried (rounds spent, paths still conflicted) and the branch
4845    /// and worktree stay exactly as they were - untouched, for a person to
4846    /// look at - which is also what makes re-entering this function
4847    /// afterwards a no-op instead of a second attempt at the same wall. A
4848    /// push failure ends the same way.
4849    async fn sync_to_base(&mut self) -> Result<()> {
4850        if self.state.status == RunStatus::AlreadyInBase {
4851            return Ok(());
4852        }
4853        self.hold_base_sync().await?;
4854        let conflicted = self
4855            .state
4856            .base_sync
4857            .as_ref()
4858            .is_some_and(|s| s.conflict.is_some());
4859        let Some(winner) = self.state.winner().cloned() else {
4860            return Ok(());
4861        };
4862
4863        let repo = self.state.repo.clone();
4864        let remote = self.state.config.merge.remote.clone();
4865        let base_branch = self.state.base_branch.clone();
4866        let tracking = format!("{remote}/{base_branch}");
4867
4868        let fetched_base =
4869            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
4870        // No network, or the remote never had this branch: the run already
4871        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4872        // otherwise), and one that got this far is not blocked by a fetch
4873        // that fails now. It just has no newer tip to compare against.
4874        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4875            return Ok(());
4876        };
4877
4878        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4879        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4880        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4881
4882        // Before any rebase, and before a recorded conflict is honoured: a
4883        // branch whose change reached the base under other commit ids has
4884        // nothing to rebase and nothing to conflict with, and a run that
4885        // already stopped on that phantom conflict recovers here on resume.
4886        // `behind == 0` with head == tip is a branch the base has since taken
4887        // in whole, whether or not a conflict was ever recorded: the ancestry
4888        // proof must still run (`classify` ignores a head still on the start
4889        // commit).
4890        if (behind > 0 || conflicted || head == tip)
4891            && self
4892                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4893                .await?
4894        {
4895            return Ok(());
4896        }
4897        if conflicted && self.state.base_recovery_pending() {
4898            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
4899            let owner =
4900                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
4901            if !fetched_base
4902                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
4903                || !git::is_clean(&winner.worktree).await?
4904                || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
4905            {
4906                return Ok(());
4907            }
4908        }
4909        let recovery = conflicted && self.state.consume_base_recovery(&head);
4910        if conflicted {
4911            // Consumption (including a refused changed HEAD) is durable before Git.
4912            self.state.save()?;
4913            if !recovery {
4914                return Ok(());
4915            }
4916        }
4917
4918        if behind == 0 {
4919            // A fixer-finished rebase moves the branch ref before the
4920            // winner's worktree is told (`sync_to_head` below). A run that
4921            // died in between resumes here with `behind == 0` and a tree still
4922            // holding the pre-rebase files, which review and the gate would
4923            // then read. That state is exactly: HEAD moved off the tip the
4924            // rebase started from, yet the tree is still identical to that
4925            // tip. A tree with edits of its own differs from it, so nothing
4926            // is thrown away.
4927            if let Some(from) = self
4928                .state
4929                .rebase_fixes
4930                .iter()
4931                .rev()
4932                .find_map(|r| r.from.clone())
4933                && from != head
4934                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4935                    .await
4936                    .is_ok_and(|o| o.ok())
4937            {
4938                git::sync_to_head(&winner.worktree).await?;
4939            }
4940            // An earlier attempt may have rebased the branch locally and died
4941            // before pushing it (only the fresh-rebase arm below pushes).
4942            // Publish it now, so the plain push at PR time is not refused as
4943            // a non-fast-forward. A run already holding a recorded conflict
4944            // never reaches here; that case is out of scope.
4945            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4946            if let Some(why) = &conflict {
4947                self.state.status = RunStatus::Blocked;
4948                self.state.event("land", why.clone());
4949            }
4950            self.state.base_sync = Some(BaseSync {
4951                tip,
4952                behind: 0,
4953                attempts: attempts + usize::from(recovery),
4954                conflict,
4955                already_in: None,
4956            });
4957            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4958                .await?;
4959            self.state.save()?;
4960            return Ok(());
4961        }
4962
4963        if attempts >= BASE_SYNC_ROUNDS && !recovery {
4964            let why = format!(
4965                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4966                 synchronization(s); synchronizing again would only race it",
4967                winner.branch
4968            );
4969            self.state.status = RunStatus::Blocked;
4970            self.state.base_sync = Some(BaseSync {
4971                tip,
4972                behind,
4973                attempts,
4974                conflict: Some(why.clone()),
4975                already_in: None,
4976            });
4977            self.state.event("land", why);
4978            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
4979                .await?;
4980            self.state.save()?;
4981            return Ok(());
4982        }
4983
4984        if self.state.foreign_checkout.is_some() {
4985            // Delivery to the operator's checkout is fast-forward only. A
4986            // rebase here would discard its saved head from our ancestry.
4987            let wait = repository_wait(&self.state, None);
4988            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
4989            let merged = git::git_raw(
4990                &winner.worktree,
4991                &[
4992                    "-c",
4993                    "core.hooksPath=/dev/null",
4994                    "merge",
4995                    "--no-ff",
4996                    "--no-autostash",
4997                    "--no-edit",
4998                    "-m",
4999                    &format!("Merge {tracking} before verifying"),
5000                    &tip,
5001                ],
5002            )
5003            .await?;
5004            let conflict = if merged.ok() {
5005                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5006                self.state.event(
5007                    "land",
5008                    format!("merged {tracking} into {} before verifying", winner.branch),
5009                );
5010                None
5011            } else {
5012                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5013                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5014                }
5015                let why = format!(
5016                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5017                    winner.branch,
5018                    merged.stderr.chars().take(600).collect::<String>()
5019                );
5020                self.state.status = RunStatus::Blocked;
5021                self.state.event("land", &why);
5022                Some(why)
5023            };
5024            let synced = conflict.is_none();
5025            self.state.base_sync = Some(BaseSync {
5026                tip,
5027                behind: if synced { 0 } else { behind },
5028                attempts: attempts + 1,
5029                conflict,
5030                already_in: None,
5031            });
5032            self.state.save()?;
5033            return Ok(());
5034        }
5035
5036        self.state.event(
5037            "land",
5038            format!(
5039                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5040                winner.branch
5041            ),
5042        );
5043        self.state.save()?;
5044
5045        // The remote's copy of the branch, read now and only if the fetch
5046        // really succeeded (a stale tracking ref must never pin a lease). It is
5047        // pushed over after a rebase only when it is a commit this branch
5048        // already contains, by ancestry or by patch (an earlier rebase of ours
5049        // that never reached the remote): anything else is somebody else's work.
5050        let branch_tracking = format!("{remote}/{}", winner.branch);
5051        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5052        let remote_tip = if self.state.task_pr_branch().is_none()
5053            && matches!(&fetched_branch, Ok(o) if o.ok())
5054        {
5055            git::rev_parse(&repo, &branch_tracking).await.ok()
5056        } else {
5057            None
5058        };
5059        // A remote tip this branch does not contain is somebody else's work:
5060        // rebasing would leave a local tip that can never be pushed. Stop
5061        // before touching anything and say so.
5062        if let Some(theirs) = &remote_tip
5063            && !git::is_ancestor(&repo, theirs, &head).await
5064            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5065                .await
5066                .is_ok_and(|missing| missing.is_empty())
5067        {
5068            let why = format!(
5069                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5070                 them",
5071                short(theirs),
5072                winner.branch
5073            );
5074            self.state.status = RunStatus::Blocked;
5075            self.state.base_sync = Some(BaseSync {
5076                tip,
5077                behind,
5078                attempts: attempts + usize::from(recovery),
5079                conflict: Some(why.clone()),
5080                already_in: None,
5081            });
5082            self.state.event("land", why);
5083            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5084                .await?;
5085            self.state.save()?;
5086            return Ok(());
5087        }
5088
5089        let scratch = self.state.dir().join("base-sync");
5090        let rebased =
5091            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5092                .await
5093            {
5094                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5095                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5096                Err(e) => Err(e),
5097            };
5098        let attempts = attempts + 1;
5099        let stop_kind = match &rebased {
5100            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5101            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5102            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5103        };
5104        match rebased {
5105            Ok(None) => {
5106                // The branch ref moved, but a worktree that already had it
5107                // checked out (the winner's) was not told; sync its index and
5108                // files before anything reads them.
5109                git::sync_to_head(&winner.worktree).await?;
5110                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5111                let mut conflict = None;
5112                if let Some(pinned) = &remote_tip {
5113                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5114                    match pushed {
5115                        Ok(o) if o.ok() => self.state.event(
5116                            "land",
5117                            format!("pushed rebased {} to {remote}", winner.branch),
5118                        ),
5119                        Ok(o) => {
5120                            conflict = Some(format!(
5121                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5122                                winner.branch,
5123                                short(pinned),
5124                                o.stderr.chars().take(600).collect::<String>()
5125                            ));
5126                        }
5127                        Err(e) => {
5128                            conflict = Some(format!(
5129                                "rebased {} locally but could not push it: {e:#}",
5130                                winner.branch
5131                            ));
5132                        }
5133                    }
5134                }
5135                if let Some(why) = &conflict {
5136                    self.state.status = RunStatus::Blocked;
5137                    self.state.event("land", why.clone());
5138                }
5139                self.state.base_sync = Some(BaseSync {
5140                    tip: tip.clone(),
5141                    behind: 0,
5142                    attempts,
5143                    conflict,
5144                    already_in: None,
5145                });
5146                self.state
5147                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
5148            }
5149            Ok(Some(conflict)) => {
5150                let why = format!(
5151                    "{} conflicts with {tracking} and did not rebase: {}",
5152                    winner.branch,
5153                    conflict.chars().take(600).collect::<String>()
5154                );
5155                self.state.status = RunStatus::Blocked;
5156                self.state.base_sync = Some(BaseSync {
5157                    tip,
5158                    behind,
5159                    attempts,
5160                    conflict: Some(why.clone()),
5161                    already_in: None,
5162                });
5163                self.state.event("land", why);
5164            }
5165            // A refused Git write or a busy repository: nothing is wrong with
5166            // the branch. The scratch rebase stays standing and the run
5167            // resumes into it (`base_sync` must not claim a conflict).
5168            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5169                self.state.save()?;
5170                return Ok(());
5171            }
5172            Err(e) => {
5173                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5174                self.state.status = RunStatus::Blocked;
5175                self.state.base_sync = Some(BaseSync {
5176                    tip,
5177                    behind,
5178                    attempts,
5179                    conflict: Some(why.clone()),
5180                    already_in: None,
5181                });
5182                self.state.event("land", why);
5183            }
5184        }
5185        if !self.state.base_sync_blocked() {
5186            self.state.base_sync_recovery.stop = None;
5187        }
5188        self.hold_base_sync_kind(stop_kind).await?;
5189        self.state.save()?;
5190        Ok(())
5191    }
5192
5193    /// Push a branch an earlier attempt rebased locally but never published,
5194    /// pinned to the remote tip read right after a successful fetch. Returns
5195    /// the reason when the run must stop; `None` when there was nothing to do
5196    /// (no remote copy, the same tip, or a remote copy this branch already
5197    /// contains, which the PR-time push fast-forwards) or the push succeeded.
5198    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5199        // Task competitions publish only after the PR text gate, to their
5200        // public head; never resurrect a candidate branch on the remote.
5201        if self.state.task_pr_branch().is_some() {
5202            return None;
5203        }
5204        let repo = self.state.repo.clone();
5205        let remote = self.state.config.merge.remote.clone();
5206        let fetched = git::fetch(&repo, &remote, branch).await;
5207        if !matches!(&fetched, Ok(o) if o.ok()) {
5208            return None;
5209        }
5210        let branch_tracking = format!("{remote}/{branch}");
5211        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5212        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5213            return None;
5214        }
5215        if !crate::reconcile::origin_missing(&repo, head, &theirs)
5216            .await
5217            .is_ok_and(|missing| missing.is_empty())
5218        {
5219            return Some(format!(
5220                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5221                 them",
5222                short(&theirs)
5223            ));
5224        }
5225        match git::push_pinned(&repo, &remote, branch, &theirs).await {
5226            Ok(o) if o.ok() => {
5227                self.state
5228                    .event("land", format!("pushed rebased {branch} to {remote}"));
5229                None
5230            }
5231            Ok(o) => Some(format!(
5232                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5233                 someone may have pushed): {}",
5234                short(&theirs),
5235                o.stderr.chars().take(600).collect::<String>()
5236            )),
5237            Err(e) => Some(format!(
5238                "{branch} is rebased locally but could not be pushed: {e:#}"
5239            )),
5240        }
5241    }
5242
5243    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
5244    /// is already on `tip` under other commit ids ([`crate::already`]); returns
5245    /// whether it did.
5246    ///
5247    /// Checked only when the base is ahead of the branch. A failing check is
5248    /// "not proven" - the ordinary rebase path then decides - never a reason to
5249    /// stop the run.
5250    ///
5251    /// The remote copy of the branch is held to the same standard as the local
5252    /// one: if it carries a tip this worktree does not, that tip must itself be
5253    /// proven in the base, or nothing is settled (a pull request would
5254    /// otherwise be closed over commits nobody checked). The pull request is
5255    /// closed *before* the terminal status is saved; if that fails for a
5256    /// reason other than a refusal (no network, a `gh` error) the run is left
5257    /// `Blocked` with the reason as its conflict, which a resume retries -
5258    /// the same recovery a phantom conflict gets.
5259    async fn settle_already_in(
5260        &mut self,
5261        branch: &str,
5262        tip: &str,
5263        head: &str,
5264        attempts: usize,
5265        behind: usize,
5266    ) -> Result<bool> {
5267        let repo = self.state.repo.clone();
5268        let remote = self.state.config.merge.remote.clone();
5269        let start = self.state.base_commit.clone();
5270        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5271            Ok(Some(e)) => e,
5272            Ok(None) => return Ok(false),
5273            Err(e) => {
5274                tracing::warn!("already-in-base check for {branch}: {e:#}");
5275                return Ok(false);
5276            }
5277        };
5278        let mut verified = vec![head.to_owned()];
5279        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5280        let fetched = git::fetch(&repo, &remote, &public_branch).await;
5281        if matches!(&fetched, Ok(o) if o.ok())
5282            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5283            && theirs != head
5284        {
5285            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5286                Ok(Some(_)) => verified.push(theirs),
5287                _ => return Ok(false),
5288            }
5289        }
5290        let base_branch = self.state.base_branch.clone();
5291        let message = format!(
5292            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5293             land",
5294            evidence.names(),
5295            evidence.proof.as_str()
5296        );
5297        let closed =
5298            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5299                .await;
5300        match closed {
5301            Ok(Ok(url)) => self
5302                .state
5303                .event("land", format!("closed {url}: superseded on {base_branch}")),
5304            Ok(Err(why)) => self
5305                .state
5306                .event("land", format!("did not close a pull request: {why}")),
5307            Err(e) => {
5308                let why = format!(
5309                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
5310                     not be closed ({e:#}); resume to retry"
5311                );
5312                self.state.status = RunStatus::Blocked;
5313                self.state.base_sync = Some(BaseSync {
5314                    tip: tip.to_owned(),
5315                    behind,
5316                    attempts,
5317                    conflict: Some(why.clone()),
5318                    already_in: None,
5319                });
5320                self.state.event("land", why);
5321                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5322                    .await?;
5323                self.state.save()?;
5324                return Ok(true);
5325            }
5326        }
5327        self.state.status = RunStatus::AlreadyInBase;
5328        self.state.base_sync_recovery.stop = None;
5329        self.state.base_sync = Some(BaseSync {
5330            tip: tip.to_owned(),
5331            behind,
5332            attempts,
5333            conflict: None,
5334            already_in: Some(evidence),
5335        });
5336        self.state.event("land", message);
5337        self.state.save()?;
5338        self.settle_questions();
5339        Ok(true)
5340    }
5341
5342    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
5343    /// last landed the winner on, once it has run, else the commit the run
5344    /// branched from.
5345    ///
5346    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
5347    /// and `vote` all happen before there is a winner to rebase, so they
5348    /// compare every candidate against the branch point on purpose, and a
5349    /// base that moves after they are already done cannot change an answer
5350    /// they already gave.
5351    fn landing_base(&self) -> String {
5352        self.state
5353            .base_sync
5354            .as_ref()
5355            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5356    }
5357
5358    // ------------------------------------------------------- operator fix
5359
5360    /// Route specific, already-recorded review findings to a fixer for a
5361    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5362    /// entry point.
5363    ///
5364    /// Distinct from `review_loop`'s own fix step in three ways: it never
5365    /// runs a reviewer wave, it never spends review-round budget, and what
5366    /// happened is recorded as an [`OperatorFixRequest`] appended to
5367    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5368    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5369    /// and vote must never be rewritten to look like a manufactured blocking
5370    /// verdict.
5371    ///
5372    /// Only meaningful once review has actually concluded: `Ready` (handed
5373    /// off with findings still open, or simply concluded clean while minor
5374    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5375    /// gate failed). Everything else is refused: a run still in progress
5376    /// should simply be resumed, and a `Merged` run's branch has already
5377    /// landed — reopening *this* run's own record cannot change that, so the
5378    /// answer there is a fresh `magi review <branch>`.
5379    ///
5380    /// A real commit here re-verifies through a fresh, ordinary review-only
5381    /// run on the same branch ([`Self::review`]) rather than reopening this
5382    /// run's own `review_loop`: once any round in this run's history went
5383    /// clean, `review_conclusion` treats that as permanent by design (the
5384    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5385    /// way to force one more genuine reviewer wave out of *this* run without
5386    /// either rewriting history or weakening that guarantee for every other
5387    /// caller. A review-only run costs nothing extra — no implementation, no
5388    /// judging, no vote — and exercises the exact same review → verify →
5389    /// gate → (human) merge path, unmodified.
5390    pub async fn fix_selected(
5391        &mut self,
5392        ids: &[String],
5393        reason: &str,
5394        allow_stale: bool,
5395    ) -> Result<()> {
5396        let reason = reason.trim();
5397        if reason.is_empty() {
5398            bail!("a fix request needs a reason — that is the operator's own record of why");
5399        }
5400        if ids.is_empty() {
5401            bail!("no finding id given");
5402        }
5403        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5404            bail!(
5405                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5406                 has already concluded — can be given a targeted fix. A run still \
5407                 in progress should simply be resumed; a `merged` run's branch has \
5408                 already landed, so its answer is a fresh `magi review <branch>`, \
5409                 not reopening this run's own record",
5410                self.state.id,
5411                self.state.status.as_str()
5412            );
5413        }
5414        let Some(winner) = self.state.winner().cloned() else {
5415            bail!("run {} has no winning candidate to fix", self.state.id);
5416        };
5417        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5418            bail!(
5419                "branch `{}` no longer exists; this run cannot be extended",
5420                winner.branch
5421            );
5422        }
5423        let home = crate::run::home();
5424        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5425            bail!(
5426                "run {} is currently being worked on by another magi process",
5427                self.state.id
5428            );
5429        }
5430        // Held for the rest of this call, including the follow-up review
5431        // below: two `magi fix` invocations against the same run must not
5432        // both reach the worktree manipulation further down, which would
5433        // otherwise race to remove and recreate the same directory — see
5434        // [`FixClaim`]'s own doc.
5435        let _claim = FixClaim::acquire(&self.state.dir())?;
5436
5437        // Resolve every id before spending anything — an unknown id refuses
5438        // the whole request rather than silently dropping it — and dedup
5439        // while keeping the operator's own order.
5440        let mut seen = BTreeSet::new();
5441        let mut findings = Vec::new();
5442        let mut missing = Vec::new();
5443        for id in ids {
5444            if !seen.insert(id.clone()) {
5445                continue;
5446            }
5447            match self.state.finding(id) {
5448                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5449                    id: f.id.clone(),
5450                    severity: f.severity,
5451                    reviewer_vote: rec.vote,
5452                    round: round.round,
5453                    round_head: round.head.clone(),
5454                    reviewer: rec.reviewer,
5455                    agent: rec.agent.clone(),
5456                    file: f.file.clone(),
5457                    line: f.line,
5458                    title: f.title.clone(),
5459                    detail: f.detail.clone(),
5460                    outcome: OperatorFixOutcome::Pending,
5461                }),
5462                None => missing.push(id.clone()),
5463            }
5464        }
5465        if !missing.is_empty() {
5466            bail!(
5467                "unknown finding id(s): {}; nothing was changed",
5468                missing.join(", ")
5469            );
5470        }
5471
5472        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5473        let stale_details: Vec<(String, String)> = findings
5474            .iter()
5475            .filter(|f| f.round_head != head_at_request)
5476            .map(|f| (f.id.clone(), f.round_head.clone()))
5477            .collect();
5478        let stale = !stale_details.is_empty();
5479        if stale && !allow_stale {
5480            bail!(
5481                "the branch has moved since some finding(s) were raised — {} — now \
5482                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5483                stale_details
5484                    .iter()
5485                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5486                    .collect::<Vec<_>>()
5487                    .join(", "),
5488                short(&head_at_request)
5489            );
5490        }
5491
5492        let request = OperatorFixRequest {
5493            requested_at: Timestamp::now(),
5494            reason: reason.to_owned(),
5495            findings,
5496            head_at_request: head_at_request.clone(),
5497            allow_stale,
5498            stale,
5499            fix: None,
5500            result_head: None,
5501            follow_up_review_run: None,
5502        };
5503        self.state.event(
5504            "fix",
5505            format!(
5506                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5507                request.findings.len(),
5508                request
5509                    .findings
5510                    .iter()
5511                    .map(|f| f.id.as_str())
5512                    .collect::<Vec<_>>()
5513                    .join(", "),
5514            ),
5515        );
5516        // Recorded now, before any worktree work or the fixer call itself —
5517        // and re-saved at each checkpoint below: a crash at any point after
5518        // this (mid fixer call, mid follow-up review) must not lose the fact
5519        // that this was requested, for which findings, and why. Everything
5520        // past this point reads and writes through `request_index` rather
5521        // than a local variable, since `request` itself is moved here.
5522        self.state.operator_fixes.push(request);
5523        self.state.save()?;
5524        let request_index = self.state.operator_fixes.len() - 1;
5525
5526        // A fresh, dedicated worktree for this one call, never the winner's
5527        // own worktree in place: that one may already be gone (folded away),
5528        // and reusing it in place would leave the branch checked out there
5529        // when the follow-up review below tries to check it out again. Freed
5530        // immediately after, either way — but only once confirmed clean:
5531        // `worktree_remove` is a `git worktree remove --force`, which would
5532        // otherwise discard uncommitted work left there by the operator or
5533        // another process before this had a chance to even look at it.
5534        let repo = self.state.repo.clone();
5535        let coordination = self.coordinate(&repo).await?;
5536        if winner.worktree.exists() {
5537            // Lockfiles a rescue commit withheld stay untracked on purpose and
5538            // are already recorded; they are not the operator's work to protect.
5539            let dirty = git::git(
5540                &winner.worktree,
5541                &["status", "--porcelain", "--untracked-files=all"],
5542            )
5543            .await?;
5544            let only_withheld = dirty.lines().all(|l| {
5545                l.strip_prefix("?? ")
5546                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5547            });
5548            if !only_withheld {
5549                bail!(
5550                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5551                     discard them first",
5552                    winner.worktree.display()
5553                );
5554            }
5555            git::worktree_remove(&self.state.repo, &winner.worktree)
5556                .await
5557                .ok();
5558        }
5559        let fix_worktree = self.state.worktree_root().join("operator-fix");
5560        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5561        git::git(
5562            &self.state.repo,
5563            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5564        )
5565        .await
5566        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5567        if !git::is_clean(&fix_worktree).await? {
5568            git::worktree_remove(&self.state.repo, &fix_worktree)
5569                .await
5570                .ok();
5571            bail!(
5572                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5573                winner.branch
5574            );
5575        }
5576        let repo_for_setup = self.state.repo.clone();
5577        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5578            Ok(()) => {
5579                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5580            }
5581            Err(e) => Err(e),
5582        };
5583        if let Err(e) = setup {
5584            git::worktree_remove(&self.state.repo, &fix_worktree)
5585                .await
5586                .ok();
5587            return Err(e);
5588        }
5589
5590        let run_id = self.state.id.clone();
5591        let prompts = self.state.config.prompts.clone();
5592        let language = self.state.config.graph.language.clone();
5593        let sessions = self.state.config.graph.sessions;
5594        let artifacts = agent::artifacts_dir(&self.state.dir());
5595        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5596            .findings
5597            .iter()
5598            .map(|f| Finding {
5599                id: f.id.clone(),
5600                severity: f.severity,
5601                file: f.file.clone(),
5602                line: f.line,
5603                title: f.title.clone(),
5604                detail: f.detail.clone(),
5605            })
5606            .collect();
5607        let fix_prompt = prompt::operator_fix_in(
5608            &self.state.instruction,
5609            &finding_list,
5610            reason,
5611            &stale_details,
5612            &head_at_request,
5613            &language,
5614            &self.state.config.graph.github_language,
5615        );
5616        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5617        drop(coordination);
5618        let (job, seat, out) = self
5619            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5620                prompt: fix_prompt.clone(),
5621                spec,
5622                seat,
5623                cwd: fix_worktree.clone(),
5624                timeout,
5625                allow_write: true,
5626                sessions,
5627                artifacts: artifacts.clone(),
5628                stem: "operator-fix".to_owned(),
5629                handover: None,
5630            })
5631            .await;
5632        let agent_id = seat.agent.clone();
5633
5634        let mut fix = FixRecord {
5635            agent: agent_id,
5636            addressed: Vec::new(),
5637            rejected: Vec::new(),
5638            notes: String::new(),
5639            committed: false,
5640            failed: None,
5641            duration_ms: 0,
5642            continuation: None,
5643        };
5644        let mut final_seat = seat.clone();
5645        match out {
5646            AgentOutcome::Ok(o) => {
5647                fix.duration_ms = o.duration_ms;
5648                let parsed = verdict::extract_json::<FixReport>(&o.text);
5649                let incomplete_reason = match &parsed {
5650                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5651                        "the reply parsed, but it reported a command whose own CLI \
5652                         never confirmed an exit status"
5653                            .to_owned(),
5654                    ),
5655                    Ok(_) => None,
5656                    Err(e) => Some(e.to_string()),
5657                };
5658                match incomplete_reason {
5659                    None => {
5660                        let report = parsed.expect("checked Ok above");
5661                        fix.addressed = report.addressed;
5662                        fix.rejected = report.rejected;
5663                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5664                    }
5665                    Some(reason) => {
5666                        let (resumed_seat, resolved, failure, cont) = self
5667                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5668                            .await;
5669                        fix.duration_ms += cont.cumulative_wait_ms;
5670                        fix.continuation = Some(cont);
5671                        final_seat = resumed_seat;
5672                        match resolved {
5673                            Some(report) => {
5674                                fix.addressed = report.addressed;
5675                                fix.rejected = report.rejected;
5676                                fix.notes =
5677                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5678                            }
5679                            None => fix.failed = failure,
5680                        }
5681                    }
5682                }
5683            }
5684            AgentOutcome::Dropped(o) => {
5685                fix.duration_ms = o.duration_ms;
5686                let why = o
5687                    .dropped
5688                    .as_ref()
5689                    .map(|d| d.why.as_str())
5690                    .unwrap_or("the CLI ended the stream without delivering its answer");
5691                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5692            }
5693            AgentOutcome::Quota(o) => {
5694                self.state.quota.push(QuotaLoss {
5695                    seat: final_seat.key.clone(),
5696                    node: "fix".to_owned(),
5697                    at: Timestamp::now(),
5698                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5699                });
5700                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5701            }
5702            AgentOutcome::Failed(e) => fix.failed = Some(e),
5703            AgentOutcome::ResourceBlocked(e) => {
5704                fix.failed = Some(format!("the fixer was not started: {e}"));
5705            }
5706        }
5707        if fix.continuation.is_none() {
5708            fix.continuation = Some(ContinuationRecord::not_needed());
5709        }
5710        self.state.seats.insert(final_seat.key.clone(), final_seat);
5711
5712        let rescue_message = format!(
5713            "magi: operator-selected fix ({}) (uncommitted work)",
5714            self.state.operator_fixes[request_index]
5715                .findings
5716                .iter()
5717                .map(|f| f.id.as_str())
5718                .collect::<Vec<_>>()
5719                .join(", ")
5720        );
5721        let repo = self.state.repo.clone();
5722        let coordination = self.coordinate(&repo).await?;
5723        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5724            self.state.note_withheld("fix", &r.withheld);
5725        }
5726        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5727        fix.committed = after != head_at_request;
5728        git::worktree_remove(&self.state.repo, &fix_worktree)
5729            .await
5730            .ok();
5731        drop(coordination);
5732
5733        self.state.event(
5734            "fix",
5735            match &fix.failed {
5736                Some(reason) => format!(
5737                    "operator fix: adoption report was lost ({reason}); {}",
5738                    if fix.committed {
5739                        "committed"
5740                    } else {
5741                        "NO new commit"
5742                    }
5743                ),
5744                None => format!(
5745                    "operator fix: {} addressed, {} rejected, {}",
5746                    fix.addressed.len(),
5747                    fix.rejected.len(),
5748                    if fix.committed {
5749                        "committed"
5750                    } else {
5751                        "NO new commit"
5752                    }
5753                ),
5754            },
5755        );
5756
5757        // Every selected finding gets an outcome — never left `Pending` once
5758        // the fixer's own turn is over. A report that never came back at all
5759        // marks every one of them `Unreported`, not silently "not addressed":
5760        // quota, a dropped stream, or an exhausted continuation are gaps in
5761        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5762        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5763        for f in &mut self.state.operator_fixes[request_index].findings {
5764            f.outcome = if fix.failed.is_some() {
5765                OperatorFixOutcome::Unreported
5766            } else if fix.addressed.contains(&f.id) {
5767                OperatorFixOutcome::Addressed
5768            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5769                OperatorFixOutcome::Rejected { why: r.why.clone() }
5770            } else {
5771                OperatorFixOutcome::Unreported
5772            };
5773        }
5774
5775        let committed = fix.committed;
5776        if committed {
5777            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5778        }
5779        self.state.operator_fixes[request_index].fix = Some(fix);
5780        // Saved again now that the fixer's own outcome is final, on top of
5781        // the save right after the request was first pushed above.
5782        self.state.save()?;
5783
5784        if committed {
5785            self.state.event(
5786                "fix",
5787                format!(
5788                    "operator fix committed {}; opening a follow-up review-only run",
5789                    short(&after)
5790                ),
5791            );
5792            // The operator asked for the fix, and the follow-up serves whatever
5793            // task the run it follows served.
5794            let origin =
5795                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5796            match Self::review(
5797                &self.state.repo,
5798                &winner.branch,
5799                self.state.config.clone(),
5800                origin,
5801            )
5802            .await
5803            {
5804                Ok(mut follow_up) => {
5805                    follow_up.state.event(
5806                        "start",
5807                        format!(
5808                            "requested by an operator fix on run {} for finding(s) {}",
5809                            self.state.id,
5810                            self.state.operator_fixes[request_index]
5811                                .findings
5812                                .iter()
5813                                .map(|f| f.id.as_str())
5814                                .collect::<Vec<_>>()
5815                                .join(", "),
5816                        ),
5817                    );
5818                    follow_up.state.save()?;
5819                    let follow_up_id = follow_up.state.id.clone();
5820                    // The follow-up is a run like any other: it belongs to the
5821                    // task of the run it follows, or to one filed for it.
5822                    let adopted = match crate::direct::adopt(
5823                        &follow_up.state,
5824                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5825                    ) {
5826                        Ok(a) => a,
5827                        Err(e) => {
5828                            // An ownerless run must not spend agent calls; it
5829                            // stays saved, and `magi run --resume` adopts it.
5830                            self.state.event(
5831                                "fix",
5832                                format!(
5833                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5834                                ),
5835                            );
5836                            self.state.save()?;
5837                            return Ok(());
5838                        }
5839                    };
5840                    let executed = follow_up.execute().await;
5841                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5842                    if let Some(a) = adopted {
5843                        a.finish(&follow_up.state, executed);
5844                    }
5845                    if let Some(e) = failure {
5846                        self.state.event(
5847                            "fix",
5848                            format!(
5849                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5850                            ),
5851                        );
5852                    }
5853                    self.state.operator_fixes[request_index].follow_up_review_run =
5854                        Some(follow_up_id);
5855                }
5856                Err(e) => {
5857                    self.state.event(
5858                        "fix",
5859                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5860                    );
5861                }
5862            }
5863            self.state.save()?;
5864        }
5865
5866        Ok(())
5867    }
5868
5869    // --------------------------------------------------------------- review
5870
5871    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5872    /// names, in order, each at most once, moving on only when the call
5873    /// advances (an error, a quota hit or nothing usable - the same decision
5874    /// point `agent::chain_advances` is for the other chained roles).
5875    ///
5876    /// Each agent gets its own seat from `Runner::seat`: the same agent
5877    /// continues its conversation (the winner's own implementer seat when it
5878    /// is the winner's author, now that the competition is over), another
5879    /// takes a fresh one so the full prompt is sent again. A handover is
5880    /// recorded under `node`, which is what makes the fallback stick for the
5881    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5882    /// is returned, so an exhausted chain reads exactly like a single failed
5883    /// fixer: one quota loss, the same wording, the same refund. An earlier
5884    /// attempt's edits are left in the tree and judged, with the final
5885    /// attempt's, by what git says afterwards.
5886    ///
5887    /// Returns the job that produced the outcome, for `continue_fix_report`.
5888    async fn ask_fixer(
5889        &mut self,
5890        winner: &Candidate,
5891        node: &'static str,
5892        round: Option<usize>,
5893        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5894    ) -> (SeatJob, SeatState, AgentOutcome) {
5895        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5896        self.ask_fixer_among(attempts, node, round, None, build)
5897            .await
5898    }
5899
5900    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5901    /// worktree and a commit: before the next agent is asked after one that
5902    /// did not satisfy the call, the worktree is put back on that commit, so
5903    /// a failed agent's partial edits are not carried into the next one.
5904    async fn ask_fixer_among(
5905        &mut self,
5906        attempts: Vec<(AgentSpec, String)>,
5907        node: &'static str,
5908        round: Option<usize>,
5909        restore: Option<(&Path, &str)>,
5910        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5911    ) -> (SeatJob, SeatState, AgentOutcome) {
5912        let run_id = self.state.id.clone();
5913        let prompts = self.state.config.prompts.clone();
5914        let cache = self.state.config.cache_dir();
5915        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5916        let mut last = None;
5917        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5918            let seat = self.seat(&key, &spec.id);
5919            let mut job = build(spec.clone(), seat);
5920            if i > 0 {
5921                job.stem = format!("{}-{}", job.stem, spec.id);
5922            }
5923            let ctx = WaveCtx {
5924                carry_seats: false,
5925                pauses: Some((&self.pause, &self.interrupt)),
5926                run: &run_id,
5927                node,
5928                prompts: &prompts,
5929                cache: cache.as_deref(),
5930                round,
5931            };
5932            let (seat, out) =
5933                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5934            // A resource block never advances the chain: the chosen agent
5935            // was not asked, so it must be the one asked again on resume.
5936            let advances = match &out {
5937                AgentOutcome::Ok(o) => agent::output_advances(o),
5938                AgentOutcome::ResourceBlocked(_) => false,
5939                _ => true,
5940            };
5941            if let Some(next) = ids.get(i + 1)
5942                && advances
5943            {
5944                self.state.seats.insert(seat.key.clone(), seat.clone());
5945                let class =
5946                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5947                let reason = match &out {
5948                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5949                    other => fail_reason(other),
5950                };
5951                // A failed agent's edits must not reach the next one, and a
5952                // restore that did not work ends the chain: the call fails
5953                // and the caller's own (confirmed) restore decides.
5954                if let Some((wt, rev)) = restore
5955                    && let Err(e) = self.graft_restore(wt, rev).await
5956                {
5957                    let why =
5958                        format!("could not restore the worktree before the next fixer: {e:#}");
5959                    tracing::warn!("{why}");
5960                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5961                        AgentOutcome::ResourceBlocked(why)
5962                    } else {
5963                        AgentOutcome::Failed(why)
5964                    };
5965                    last = Some((job, seat, out));
5966                    break;
5967                }
5968                record_handover(
5969                    &mut self.state,
5970                    node,
5971                    &seat.key,
5972                    &spec.id,
5973                    next,
5974                    &class,
5975                    &reason,
5976                );
5977                continue;
5978            }
5979            last = Some((job, seat, out));
5980            break;
5981        }
5982        last.expect("the fixer chain always has an entry")
5983    }
5984
5985    /// The one repair a verification-only failure earns (see
5986    /// [`crate::recovery`]), before the first review round of a run that took
5987    /// a branch over.
5988    ///
5989    /// Saved as started *before* the fixer is called, so a resume never
5990    /// repairs twice; a start found on re-entry is rolled back to the commit
5991    /// it began from. A failed or unusable fixer leaves the tree as it was and
5992    /// the review goes on over it, so the first round's own verification
5993    /// reports the failure again. The failure is never carried into a round as
5994    /// a result: verification is re-run on whatever commit this leaves.
5995    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5996        use crate::recovery::{Mode, Repair};
5997        if !self.state.reviews.is_empty() {
5998            return Ok(());
5999        }
6000        let Some(mut rec) = self.state.recovery.clone() else {
6001            return Ok(());
6002        };
6003        if rec.mode != Mode::VerifyRepair {
6004            return Ok(());
6005        }
6006        let wt = winner.worktree.clone();
6007        let mut head = git::rev_parse(&wt, "HEAD").await?;
6008        match rec.repair.clone() {
6009            Repair::Done { .. } => return Ok(()),
6010            Repair::WaitingForRestore { before } => {
6011                self.graft_restore(&wt, &before).await?;
6012                head = before;
6013                rec.repair = Repair::NotStarted;
6014                self.state.recovery = Some(rec.clone());
6015                self.state.save()?;
6016            }
6017            Repair::Started { before } => {
6018                // Interrupted: the unfinished edits are not trusted.
6019                // An unconfirmed rollback blocks the run with `Started` kept,
6020                // so a resume retries it; the tree is never reviewed as is.
6021                if let Err(e) = self.graft_restore(&wt, &before).await {
6022                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6023                        return Err(e);
6024                    }
6025                    return self
6026                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6027                        .await;
6028                }
6029                rec.repair = Repair::Done {
6030                    before: before.clone(),
6031                    after: before,
6032                    committed: false,
6033                    failed: Some("interrupted; rolled back".to_owned()),
6034                };
6035                self.state
6036                    .event("recovery", "the repair was interrupted and rolled back");
6037                self.state.recovery = Some(rec);
6038                self.state.save()?;
6039                return Ok(());
6040            }
6041            Repair::NotStarted => {}
6042        }
6043        if crate::recovery::recheck(&mut rec, &head) {
6044            self.state.event("recovery", rec.summary());
6045            self.state.recovery = Some(rec);
6046            self.state.save()?;
6047            return Ok(());
6048        }
6049        rec.repair = Repair::Started {
6050            before: head.clone(),
6051        };
6052        self.state.recovery = Some(rec.clone());
6053        self.state.save()?;
6054
6055        let language = self.state.config.graph.language.clone();
6056        let sessions = self.state.config.graph.sessions;
6057        let artifacts = agent::artifacts_dir(&self.state.dir());
6058        let fix_prompt = prompt::recovery_fix(
6059            &self.state.instruction,
6060            &rec,
6061            self.state.config.graph.review_rounds,
6062            &language,
6063        );
6064        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6065        self.state.event(
6066            "recovery",
6067            format!(
6068                "repair: the fixer starts from the failed verification on {}",
6069                short(&head)
6070            ),
6071        );
6072        let (_job, seat, out) = self
6073            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6074                prompt: fix_prompt.clone(),
6075                spec,
6076                seat,
6077                cwd: wt.clone(),
6078                timeout,
6079                allow_write: true,
6080                sessions,
6081                artifacts: artifacts.clone(),
6082                stem: "fix-recovery".to_owned(),
6083                handover: None,
6084            })
6085            .await;
6086        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6087            rec.repair = Repair::WaitingForRestore { before: head };
6088            self.state.recovery = Some(rec);
6089            return self.stop_for_resource();
6090        }
6091        let failed = match &out {
6092            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6093            AgentOutcome::Quota(o) => {
6094                self.state.quota.push(QuotaLoss {
6095                    seat: seat.key.clone(),
6096                    node: "fix".to_owned(),
6097                    at: Timestamp::now(),
6098                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6099                });
6100                Some("rate limited (quota); the fixer could not run".to_owned())
6101            }
6102            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6103            AgentOutcome::Failed(e) => Some(e.clone()),
6104        };
6105        self.state.seats.insert(seat.key.clone(), seat);
6106        let coordination = self.coordinate(&wt).await?;
6107        let failed = match failed {
6108            // The agent's partial edits are not kept.
6109            Some(why) => {
6110                if let Err(e) = git::restore_to(&wt, &head).await {
6111                    return self
6112                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6113                        .await;
6114                }
6115                Some(why)
6116            }
6117            None => {
6118                if let Ok(r) = git::rescue_commit(
6119                    &wt,
6120                    "magi: recovery repair of the failed verification (uncommitted work)",
6121                )
6122                .await
6123                {
6124                    self.state.note_withheld("fix", &r.withheld);
6125                }
6126                None
6127            }
6128        };
6129        let after = git::rev_parse(&wt, "HEAD").await?;
6130        drop(coordination);
6131        self.state.event(
6132            "recovery",
6133            match &failed {
6134                Some(why) => format!("repair: nothing kept ({why})"),
6135                None if after != head => {
6136                    format!("repair: committed {}", short(&after))
6137                }
6138                None => "repair: the fixer made no commit".to_owned(),
6139            },
6140        );
6141        rec.repair = Repair::Done {
6142            before: head.clone(),
6143            committed: after != head,
6144            after,
6145            failed,
6146        };
6147        self.state.recovery = Some(rec);
6148        self.state.save()?;
6149        Ok(())
6150    }
6151
6152    /// A repair whose rollback could not be confirmed: the run is blocked and
6153    /// resumable, and the unconfirmed state is never reviewed.
6154    async fn stop_recovery(
6155        &mut self,
6156        mut rec: crate::recovery::RecoveryRecord,
6157        head: &str,
6158        why: String,
6159    ) -> Result<()> {
6160        // `Started` stays recorded: a resume retries the restore.
6161        rec.repair = crate::recovery::Repair::Started {
6162            before: head.to_owned(),
6163        };
6164        self.state.recovery = Some(rec);
6165        self.state
6166            .event("recovery", format!("repair could not be undone: {why}"));
6167        self.state.status = RunStatus::Blocked;
6168        self.state.save()?;
6169        bail!("recovery repair could not be rolled back: {why}")
6170    }
6171
6172    async fn review_loop(&mut self) -> Result<()> {
6173        if self.hold_base_sync().await? {
6174            return Ok(());
6175        }
6176        loop {
6177            if self
6178                .state
6179                .base_sync
6180                .as_ref()
6181                .is_some_and(|s| s.conflict.is_some())
6182            {
6183                return Ok(());
6184            }
6185            if self.catchup_repairable() {
6186                let record = self.state.reviews.last().expect("repairable round").clone();
6187                let winner = self
6188                    .state
6189                    .winner()
6190                    .cloned()
6191                    .context("catch-up repair has no winner")?;
6192                if !self.fix_review_round(&winner, record).await? {
6193                    return Ok(());
6194                }
6195            }
6196            self.review_loop_pass().await?;
6197            if !self.catchup_repairable() {
6198                return Ok(());
6199            }
6200        }
6201    }
6202
6203    /// Derive an owed catch-up repair from durable round evidence, including
6204    /// reentry after a crash before the fixer was dispatched.
6205    fn catchup_repairable(&self) -> bool {
6206        self.state.reviews.len() < self.state.config.graph.review_rounds
6207            && self.state.reviews.last().is_some_and(|r| {
6208                !r.clean
6209                    && r.blocking == 0
6210                    && r.fix.is_none()
6211                    && r.e2e_status() == E2eStatus::Failed
6212            })
6213    }
6214
6215    async fn review_loop_pass(&mut self) -> Result<()> {
6216        // A base that would not rebase is a person's decision, not a review
6217        // round: nothing here would change the answer, and reviewers and a
6218        // fixer would be spending real budget on a tree that cannot land
6219        // regardless of what they find.
6220        if self
6221            .state
6222            .base_sync
6223            .as_ref()
6224            .is_some_and(|s| s.conflict.is_some())
6225        {
6226            return Ok(());
6227        }
6228        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
6229        // agent files with `magi task add` name the run that paid for it. The
6230        // prompt overlay is cloned alongside it because the waves borrow it
6231        // while `self` is mutably borrowed by the node's own bookkeeping.
6232        let run_id = self.state.id.clone();
6233        let prompts = self.state.config.prompts.clone();
6234        let Some(winner) = self.state.winner().cloned() else {
6235            return Ok(());
6236        };
6237        let max_rounds = self.state.config.graph.review_rounds;
6238        // A clean round, an exhausted round budget, or a stalled tree (see
6239        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
6240        // they are recorded — recomputed here, not read off `status`, so a
6241        // reentry into a run that already stopped restates the identical
6242        // verdict instead of silently handing back whatever an earlier node
6243        // in this same walk clobbered `status` to (a solo-candidate
6244        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
6245        // below runs an empty range once the budget is spent, and would
6246        // otherwise fall through without touching `status` at all.
6247        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
6248            // A reentry after a crash between the last round's save and
6249            // `stop_reviewing` reaches the hand-off here, not there.
6250            if status == RunStatus::Gating {
6251                self.record_contested_handoff();
6252            }
6253            self.state.status = status;
6254            self.state.save()?;
6255            return Ok(());
6256        }
6257        self.state.status = RunStatus::Reviewing;
6258        // A last recorded round whose own verification never resolved
6259        // (`ResourceBlocked` — the shared build cache, not the patch) is
6260        // never a concluded round, whatever the round budget says: starting
6261        // a fresh round on top of it would spend a whole new reviewer wave
6262        // re-reading an unchanged patch instead of just retrying the one
6263        // check that actually needs it, and once the budget is spent the
6264        // loop below has nothing left to do at all (its range is empty).
6265        // Retry that check directly instead, exactly the same retry
6266        // `stop_reviewing` already does for its own catch-up case.
6267        if self.state.reviews.last().is_some_and(|r| {
6268            r.e2e_status() == E2eStatus::ResourceBlocked
6269                || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6270        }) {
6271            let shell = self.state.config.shell();
6272            return self
6273                .stop_unresolved_review(
6274                    "the last round's own verification never resolved",
6275                    &shell,
6276                    &winner.worktree,
6277                )
6278                .await;
6279        }
6280
6281        self.recovery_repair(&winner).await?;
6282        if self.resource_halt()? {
6283            return Ok(());
6284        }
6285
6286        let repo = self.state.repo.clone();
6287        let root = self.state.worktree_root();
6288        let language = self.state.config.graph.language.clone();
6289        let sessions = self.state.config.graph.sessions;
6290        let artifacts = self.merge_repair_artifacts();
6291        let base = self.landing_base();
6292        let base_short = short(&base);
6293        let reviewers = self.roles.reviewers.clone();
6294        let shell = self.state.config.shell();
6295
6296        for round in (self.state.reviews.len() + 1)..=max_rounds {
6297            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6298            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6299            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6300            // The prior round's own record, already persisted — never a
6301            // hand-carried variable of just its failing output: that is
6302            // exactly what let a round's e2e result drift out of sync with
6303            // which commit it was actually about (see `SCHEMA`'s doc for
6304            // schema 8). Judged against `head`, the commit reviewers are
6305            // about to look at now, so the summary always reads as "an
6306            // earlier head" here — this round's own patch has not been
6307            // checked yet.
6308            let prev_verification = self
6309                .state
6310                .reviews
6311                .last()
6312                .and_then(|r| r.verification_summary(&head));
6313            // Only the first round of a run that took a branch over: later
6314            // rounds have this run's own history.
6315            let prior_section = match (&self.state.recovery, round) {
6316                (Some(rec), 1) if self.state.reviews.is_empty() => {
6317                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6318                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6319                            Some(crate::recovery::head_of(
6320                                &git::diff(&winner.worktree, reviewed, "HEAD")
6321                                    .await
6322                                    .unwrap_or_default(),
6323                                crate::recovery::SINCE_DIFF_MAX,
6324                            ))
6325                        }
6326                        _ => None,
6327                    };
6328                    Some(prompt::recovery_review(rec, since.as_deref()))
6329                }
6330                _ => None,
6331            };
6332
6333            // Each reviewer gets its own detached checkout of exactly this
6334            // commit: nobody can perturb the winner's tree, and the fixer can
6335            // keep working without racing a reviewer.
6336            let mut jobs = Vec::new();
6337            let coordination = self.coordinate(&repo).await?;
6338            for (r, spec) in reviewers.iter().cloned().enumerate() {
6339                let wt = root.join(format!("review-{}", r + 1));
6340                // `clean -fdx` in a reset wipes setup's products, so every
6341                // round sets the seat up again.
6342                if wt.exists() {
6343                    git::reset_detached(&wt, &head).await?;
6344                } else {
6345                    git::worktree_add_detached(&repo, &wt, &head).await?;
6346                }
6347                self.setup_seat_worktree(&repo, &wt, false).await?;
6348                let seat_key = format!("review-{}", r + 1);
6349                // The seat starts the round on whoever answered it last, not
6350                // on the agent the spec names, so a failure is not re-paid.
6351                let spec = pick_start_spec(
6352                    &self.roles.reviewer_roster,
6353                    spec,
6354                    self.state.seat_history.get(&seat_key),
6355                );
6356                let seat = self.seat(&seat_key, &spec.id);
6357                jobs.push(SeatJob {
6358                    prompt: prompt::review(&prompt::ReviewCtx {
6359                        instruction: &self.state.instruction,
6360                        branch: &winner.branch,
6361                        base_short: &base_short,
6362                        stat: &stat,
6363                        patch: &patch,
6364                        verification: prev_verification.as_ref(),
6365                        prior: prior_section.as_deref(),
6366                        reviewers: reviewers.len(),
6367                        round,
6368                        rounds: max_rounds,
6369                        // A review-only run has no rankings, so nothing
6370                        // competed for this patch and the reviewer is told so.
6371                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6372                        lens: Lens::for_seat(r),
6373                        language: &language,
6374                        github_language: &self.state.config.graph.github_language,
6375                    }),
6376                    spec,
6377                    seat,
6378                    cwd: wt,
6379                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6380                    allow_write: false,
6381                    sessions,
6382                    artifacts: artifacts.clone(),
6383                    stem: format!("review-{round}-{}", r + 1),
6384                    handover: None,
6385                });
6386            }
6387
6388            drop(coordination);
6389            self.state.event(
6390                "review",
6391                format!(
6392                    "round {round}: {} reviewers on {}",
6393                    jobs.len(),
6394                    short(&head)
6395                ),
6396            );
6397            let mut quota_losses = Vec::new();
6398            let review_retries = self.state.config.graph.retries;
6399            let review_cache = self.state.config.cache_dir();
6400            let ctx = WaveCtx {
6401                carry_seats: true,
6402                pauses: Some((&self.pause, &self.interrupt)),
6403                run: &run_id,
6404                node: "review",
6405                prompts: &prompts,
6406                cache: review_cache.as_deref(),
6407                round: Some(round),
6408            };
6409            let results = ask_json_wave::<Review>(
6410                jobs,
6411                Arc::clone(&self.sem),
6412                review_retries,
6413                &self.roles.reviewer_roster,
6414                &ctx,
6415                &mut quota_losses,
6416                &mut self.state,
6417                &|_: &Review| Ok(()),
6418            )
6419            .await;
6420            if self.state.resource_block.is_some() {
6421                return self.stop_for_resource();
6422            }
6423            // Counted before the move below: how many of *this* round's
6424            // reviewer seats were lost to their own rate limit, as opposed to
6425            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6426            let round_quota_missing = quota_losses.len();
6427            self.state.quota.extend(quota_losses);
6428
6429            let mut records = Vec::new();
6430            let mut all_findings = Vec::new();
6431            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6432                let agent_id = seat.agent.clone();
6433                self.state.seats.insert(seat.key.clone(), seat);
6434                let mut record = ReviewRecord {
6435                    reviewer: r + 1,
6436                    agent: agent_id,
6437                    summary: String::new(),
6438                    findings: Vec::new(),
6439                    vote: None,
6440                    failed: None,
6441                    duration_ms: 0,
6442                    // Set for both outcomes: `failed: Some(_)` with
6443                    // `attempts > 0` is a seat every retry still lost, not a
6444                    // recovered one — only `failed: None` with `attempts > 0`
6445                    // reads as "answered after a nudge" (see this field's own
6446                    // doc).
6447                    attempts,
6448                };
6449                match res {
6450                    Ok((review, out)) => {
6451                        // Sanitized here, at the point every other piece of
6452                        // agent prose in this file is (candidate summaries,
6453                        // deliberation turns, vote reasons): a reviewer's own
6454                        // words are the one thing about it that could name
6455                        // it, and reconsideration below broadcasts this same
6456                        // summary and these same findings to every other
6457                        // seat on the panel.
6458                        record.summary =
6459                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6460                        record.vote = Some(review.vote);
6461                        record.duration_ms = out.duration_ms;
6462                        for (n, mut f) in review.findings.into_iter().enumerate() {
6463                            // ids are magi's, never the agent's: the fixer's
6464                            // adoption report is keyed by them.
6465                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6466                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6467                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6468                            // `file` is agent-supplied prose too, never
6469                            // checked against the real tree — the same
6470                            // exposure `title`/`detail` above have, just in
6471                            // a field easy to forget because it looks like a
6472                            // path rather than free text.
6473                            f.file = f
6474                                .file
6475                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6476                            all_findings.push(f.clone());
6477                            record.findings.push(f);
6478                        }
6479                        self.state.event(
6480                            "review",
6481                            format!(
6482                                "round {round}: reviewer {} voted {} with {} finding(s)",
6483                                r + 1,
6484                                review.vote.label(),
6485                                record.findings.len()
6486                            ),
6487                        );
6488                    }
6489                    Err(e) => {
6490                        record.failed = Some(e.to_string());
6491                        self.state.event(
6492                            "review",
6493                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6494                        );
6495                    }
6496                }
6497                records.push(record);
6498            }
6499
6500            // Tally the round's votes and, if they split, spend the one
6501            // round of reconsideration the split -> deliberate -> revote
6502            // shape `judge`/`vote` use for the panel, sized down to what a
6503            // read-only review round can afford: one round, and a revote
6504            // rather than an argument, because the panel already wrote its
6505            // reasoning down as findings the first time around.
6506            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6507            let vote_split =
6508                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6509            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6510            if vote_split {
6511                self.state.event(
6512                    "review",
6513                    format!(
6514                        "round {round}: votes split ({}) — one round of reconsideration",
6515                        initial_votes
6516                            .iter()
6517                            .map(|v| v.label())
6518                            .collect::<Vec<_>>()
6519                            .join(", ")
6520                    ),
6521                );
6522                // Seats read every seat's findings and votes, still numbered
6523                // and never named — the same anonymity `review` itself keeps.
6524                let panel: Vec<ReviewSeatReport<'_>> = records
6525                    .iter()
6526                    .filter_map(|r| {
6527                        r.vote.map(|vote| ReviewSeatReport {
6528                            reviewer: r.reviewer,
6529                            vote,
6530                            summary: &r.summary,
6531                            findings: &r.findings,
6532                        })
6533                    })
6534                    .collect();
6535
6536                let mut jobs = Vec::new();
6537                let mut seats_at = Vec::new();
6538                for (r, spec) in reviewers.iter().cloned().enumerate() {
6539                    // A seat with no initial vote has nothing to reconsider
6540                    // from and stays absent, the same as it stayed absent
6541                    // from `panel` above.
6542                    if records[r].vote.is_none() {
6543                        continue;
6544                    }
6545                    let wt = root.join(format!("review-{}", r + 1));
6546                    let seat_key = format!("review-{}", r + 1);
6547                    let spec = self.occupant(&seat_key, spec);
6548                    let seat = self.seat(&seat_key, &spec.id);
6549                    // A seat with no live session has already forgotten the
6550                    // initial review's prompt — restate the patch it is
6551                    // voting on, the same as `deliberate`/`vote` do for a
6552                    // judge in the same position.
6553                    // The panel already carries this seat's own review and
6554                    // vote, so restating the patch makes the prompt whole for
6555                    // a seat handed to another agent.
6556                    let build = |with_patch: bool| {
6557                        prompt::review_reconsider(&ReviewReconsiderCtx {
6558                            instruction: &self.state.instruction,
6559                            reviewer: r + 1,
6560                            lens: Lens::for_seat(r),
6561                            panel: &panel,
6562                            patch: with_patch.then_some(ReviewPatch {
6563                                branch: &winner.branch,
6564                                base_short: &base_short,
6565                                stat: &stat,
6566                                patch: &patch,
6567                            }),
6568                            round,
6569                            rounds: max_rounds,
6570                            language: &language,
6571                        })
6572                    };
6573                    let full = build(true);
6574                    let prompt = if has_context(&spec, &seat, sessions) {
6575                        build(false)
6576                    } else {
6577                        full.clone()
6578                    };
6579                    jobs.push(SeatJob {
6580                        prompt,
6581                        spec,
6582                        seat,
6583                        cwd: wt,
6584                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6585                        allow_write: false,
6586                        sessions,
6587                        artifacts: artifacts.clone(),
6588                        stem: format!("review-{round}-reconsider-{}", r + 1),
6589                        handover: Some(full),
6590                    });
6591                    seats_at.push(r);
6592                }
6593
6594                let mut recon_quota_losses = Vec::new();
6595                let recon_cache = self.state.config.cache_dir();
6596                let recon_ctx = WaveCtx {
6597                    carry_seats: true,
6598                    pauses: Some((&self.pause, &self.interrupt)),
6599                    run: &run_id,
6600                    node: "review",
6601                    prompts: &prompts,
6602                    cache: recon_cache.as_deref(),
6603                    round: Some(round),
6604                };
6605                let recon_results = ask_json_wave::<ReviewRevote>(
6606                    jobs,
6607                    Arc::clone(&self.sem),
6608                    review_retries,
6609                    &self.roles.reviewer_roster,
6610                    &recon_ctx,
6611                    &mut recon_quota_losses,
6612                    &mut self.state,
6613                    &|_: &ReviewRevote| Ok(()),
6614                )
6615                .await;
6616                if self.state.resource_block.is_some() {
6617                    return self.stop_for_resource();
6618                }
6619                self.state.quota.extend(recon_quota_losses);
6620
6621                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6622                    let agent_id = seat.agent.clone();
6623                    self.state.seats.insert(seat.key.clone(), seat);
6624                    let mut rec = ReviewRevoteRecord {
6625                        reviewer: r + 1,
6626                        agent: agent_id,
6627                        vote: None,
6628                        reason: String::new(),
6629                        failed: None,
6630                    };
6631                    match res {
6632                        Ok((rv, _)) => {
6633                            rec.vote = Some(rv.vote);
6634                            rec.reason =
6635                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6636                            self.state.event(
6637                                "review",
6638                                format!(
6639                                    "round {round}: reviewer {} revoted {}",
6640                                    r + 1,
6641                                    rv.vote.label()
6642                                ),
6643                            );
6644                        }
6645                        Err(e) => {
6646                            rec.failed = Some(e.to_string());
6647                            self.state.event(
6648                                "review",
6649                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6650                            );
6651                        }
6652                    }
6653                    reconsideration.push(rec);
6654                }
6655            } else if initial_votes.len() > 1 {
6656                self.state.event(
6657                    "review",
6658                    format!(
6659                        "round {round}: votes agreed ({}) — no reconsideration",
6660                        initial_votes[0].label()
6661                    ),
6662                );
6663            }
6664
6665            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6666            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6667            // A round that already has a blocking finding and a round left to
6668            // try is going back to the fixer no matter what `verify.e2e`
6669            // says, so running it first only spends the loop's slowest step
6670            // (minutes, for a Rust repo's full test suite) on a head about
6671            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6672            // runs once a round has no blocking findings left (see
6673            // `round_is_clean`, which a deferred — empty — `e2e` can never
6674            // satisfy since `blocking` is nonzero whenever this branch is
6675            // taken), and `stop_reviewing` forces a real run before it will
6676            // ever read a deferred round as green.
6677            let defer_e2e =
6678                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6679            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6680                let reason =
6681                    format!("{blocking} blocking finding(s) already required a fix this round");
6682                self.state.event(
6683                    "verify",
6684                    format!(
6685                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6686                         {}); it will run once a round has none left",
6687                        short(&head)
6688                    ),
6689                );
6690                (Vec::new(), false, true, Some(reason))
6691            } else {
6692                let e2e_commands = self.state.config.verify.e2e.clone();
6693                let cache_dir = self.state.config.cache_dir();
6694                let context = format!("round {round}");
6695                let wait = self.repository_wait();
6696                let (e2e, verify_retried) = with_cache_lease(
6697                    &mut self.state,
6698                    cache_dir.as_deref(),
6699                    "e2e",
6700                    "e2e",
6701                    &winner.worktree,
6702                    &head,
6703                    verify_timeout,
6704                    &context,
6705                    &wait,
6706                    |state, budget| {
6707                        let shell = shell.clone();
6708                        let e2e_commands = e2e_commands.clone();
6709                        let worktree = winner.worktree.clone();
6710                        let context = context.clone();
6711                        async move {
6712                            run_e2e_with_retry(
6713                                state,
6714                                &shell,
6715                                &e2e_commands,
6716                                &worktree,
6717                                budget,
6718                                &context,
6719                            )
6720                            .await
6721                        }
6722                    },
6723                )
6724                .await?;
6725                (e2e, verify_retried, false, None)
6726            };
6727
6728            let expected = records.len();
6729            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6730            let incomplete = answered < expected;
6731            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6732            let policy = self.state.config.graph.incomplete_review;
6733            let clean = round_is_clean(
6734                blocking,
6735                e2e_ok,
6736                answered,
6737                expected,
6738                round_quota_missing,
6739                policy,
6740            );
6741
6742            let mut round_record = ReviewRound {
6743                round,
6744                head: head.clone(),
6745                verified_head: None,
6746                verified_at: None,
6747                reviews: records,
6748                e2e,
6749                verify_retried,
6750                e2e_deferred,
6751                e2e_defer_reason,
6752                fix: None,
6753                blocking,
6754                answered,
6755                expected,
6756                clean,
6757                progressed: false,
6758                vote_split,
6759                reconsideration,
6760                verdict: None,
6761            };
6762            // The final vote per seat is its revote where reconsideration
6763            // ran and answered, its initial vote otherwise — the same
6764            // fallback `tally` uses for a judge whose private vote failed.
6765            round_record.verdict = ReviewVote::worst(
6766                round_record
6767                    .final_votes()
6768                    .into_iter()
6769                    .map(|(_, _, vote)| vote),
6770            );
6771            // Which commit and when magi actually attempted to check —
6772            // known the moment a command was dispatched against `head`,
6773            // whether or not it finished: a resource-blocked attempt still
6774            // targeted a specific commit at a specific time, and leaving
6775            // that unrecorded is exactly what made `verification_summary`
6776            // report a fresh attempt as "commit unknown ... recorded before
6777            // this was tracked", indistinguishable from a genuinely old,
6778            // untracked record. Only a deferred or unconfigured round never
6779            // ran at all and has nothing to record — see
6780            // `ReviewRound::verified_head`'s own doc.
6781            if !matches!(
6782                round_record.e2e_status(),
6783                E2eStatus::Deferred | E2eStatus::NotConfigured
6784            ) {
6785                round_record.verified_head = Some(head.clone());
6786                round_record.verified_at = Some(Timestamp::now());
6787            }
6788
6789            if incomplete {
6790                let missing: Vec<String> = round_record
6791                    .reviews
6792                    .iter()
6793                    .filter(|r| r.failed.is_some())
6794                    .map(|r| format!("review-{}", r.reviewer))
6795                    .collect();
6796                self.state.event(
6797                    "review",
6798                    format!(
6799                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6800                        missing.join(", ")
6801                    ),
6802                );
6803            }
6804
6805            if clean {
6806                self.state.event(
6807                    "review",
6808                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6809                        format!(
6810                            "round {round}: clean (warn policy, incomplete panel) — no \
6811                             blocking findings from the seats that answered, verification green"
6812                        )
6813                    } else if incomplete {
6814                        format!(
6815                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6816                             quorum) — no blocking findings from the seats that answered, \
6817                             verification green",
6818                            expected - answered
6819                        )
6820                    } else {
6821                        format!("round {round}: clean — no blocking findings, verification green")
6822                    },
6823                );
6824                self.state.reviews.push(round_record);
6825                self.state.status = RunStatus::Gating;
6826                self.state.save()?;
6827                return Ok(());
6828            }
6829
6830            // Nothing was raised and verification passed, but not every seat
6831            // answered and `round_is_clean` still refused to call it clean —
6832            // either a seat is missing for a reason other than its own quota
6833            // (a crash, a timeout, unparsable output — worth another try), or
6834            // every seat that could have answered lost its quota and nobody
6835            // is left to decide on: re-review rather than send the fixer
6836            // after a round with nothing to fix.
6837            if incomplete && blocking == 0 && e2e_ok {
6838                self.state.reviews.push(round_record);
6839                self.state.save()?;
6840                if round == max_rounds {
6841                    self.state.status = RunStatus::Blocked;
6842                    self.state.event(
6843                        "review",
6844                        format!(
6845                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6846                             refusing to call it clean",
6847                            expected - answered
6848                        ),
6849                    );
6850                    return Ok(());
6851                }
6852                continue;
6853            }
6854
6855            // Nothing for the fixer to act on (`blocking == 0`) and the only
6856            // reason this round is not clean is that magi itself never got
6857            // a command to run — the shared build cache, not the patch (see
6858            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6859            // the fixer would invite a change to appease contention that has
6860            // nothing to do with the diff, and would leave this attempt
6861            // sitting in the next round's prompt as if it were about an
6862            // earlier, superseded commit rather than what it actually is:
6863            // the same head, still waiting to be checked. Wait for it the
6864            // same way the final round's own contention is already handled,
6865            // whatever round this happens to be.
6866            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6867                self.state.reviews.push(round_record);
6868                return self
6869                    .stop_unresolved_review(
6870                        "the round's own verification could not run",
6871                        &shell,
6872                        &winner.worktree,
6873                    )
6874                    .await;
6875            }
6876
6877            if round == max_rounds {
6878                self.state.reviews.push(round_record);
6879                return self
6880                    .stop_reviewing(
6881                        &format!(
6882                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6883                        ),
6884                        &shell,
6885                        &winner.worktree,
6886                    )
6887                    .await;
6888            }
6889
6890            if !self.fix_review_round(&winner, round_record).await? {
6891                return Ok(());
6892            }
6893        }
6894        Ok(())
6895    }
6896
6897    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
6898    /// False means admission or continuation policy stopped the review loop.
6899    async fn fix_review_round(
6900        &mut self,
6901        winner: &Candidate,
6902        mut round_record: ReviewRound,
6903    ) -> Result<bool> {
6904        let round = round_record.round;
6905        let shell = self.state.config.shell();
6906        let step = self.run_fix_round(winner, &mut round_record).await?;
6907        if matches!(step, FixStep::Resource) {
6908            self.stop_for_resource()?;
6909            return Ok(false);
6910        }
6911        if self.state.reviews.len() < round {
6912            self.state.reviews.push(round_record);
6913        } else {
6914            self.state.reviews[round - 1] = round_record;
6915        }
6916        self.state.save()?;
6917
6918        // The fixer's own report never came back this round, even after
6919        // `continue_fix_report`'s own budget was spent on it — not an
6920        // ordinary "no report" (dropped stream, quota, plain failure),
6921        // which already reads that way and is left to the existing round
6922        // budget. Stopping here, rather than opening another round, is
6923        // what keeps a next reviewer/fixer wave from ever being
6924        // dispatched onto `winner.worktree` while whatever the seat's
6925        // last call may still have running there is unaccounted for: no
6926        // process liveness check exists (and none is being added — see
6927        // AGENTS.md/this task's own scope), so the only way to honour
6928        // "nothing starts before a valid report returns" is to not start
6929        // anything further on this worktree from this run at all.
6930        if matches!(step, FixStep::Recorded { report_lost: true }) {
6931            self.stop_reviewing(
6932                "the fixer's adoption report never came back, even after resuming its \
6933                         own seat; refusing to start another round against the same worktree \
6934                         while that is unresolved",
6935                &shell,
6936                &winner.worktree,
6937            )
6938            .await?;
6939            return Ok(false);
6940        }
6941
6942        let streak = self
6943            .state
6944            .reviews
6945            .iter()
6946            .rev()
6947            .take_while(|r| !r.progressed)
6948            .count();
6949        if streak >= STAGNANT_LIMIT {
6950            self.stop_reviewing(
6951                &format!("the tree has not moved against base for {streak} round(s) in a row"),
6952                &shell,
6953                &winner.worktree,
6954            )
6955            .await?;
6956            return Ok(false);
6957        }
6958        Ok(true)
6959    }
6960
6961    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
6962    async fn run_fix_round(
6963        &mut self,
6964        winner: &Candidate,
6965        round_record: &mut ReviewRound,
6966    ) -> Result<FixStep> {
6967        let round = round_record.round;
6968        let max_rounds = self.state.config.graph.review_rounds;
6969        let run_id = self.state.id.clone();
6970        let prompts = self.state.config.prompts.clone();
6971        let language = self.state.config.graph.language.clone();
6972        let sessions = self.state.config.graph.sessions;
6973        let artifacts = self.merge_repair_artifacts();
6974        let base = self.landing_base();
6975        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6976        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6977        let this_round_verification = round_record.verification_summary(&head);
6978        let all_findings: Vec<_> = round_record
6979            .reviews
6980            .iter()
6981            .flat_map(|r| r.findings.iter().cloned())
6982            .collect();
6983        // Fix. The winner's own implementer seat continues its conversation:
6984        // the competition is over, so context is pure benefit now.
6985        let blocking_findings: Vec<_> = all_findings
6986            .iter()
6987            .filter(|f| f.severity.blocks())
6988            .cloned()
6989            .collect();
6990        let fix_prompt = prompt::fix_in(
6991            &self.state.instruction,
6992            &blocking_findings,
6993            this_round_verification.as_ref(),
6994            round,
6995            max_rounds,
6996            &language,
6997            &self.state.config.graph.github_language,
6998        );
6999        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7000        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7001        let (job, seat, out) = self
7002            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7003                prompt: fix_prompt.clone(),
7004                spec,
7005                seat,
7006                cwd: winner.worktree.clone(),
7007                timeout,
7008                allow_write: true,
7009                sessions,
7010                artifacts: artifacts.clone(),
7011                stem: format!("fix-{round}"),
7012                handover: None,
7013            })
7014            .await;
7015        // Not asked: the round is not recorded (so a resume repeats it),
7016        // no fixer was handed over to, and the seat is left as it was.
7017        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7018            return Ok(FixStep::Resource);
7019        }
7020        let agent_id = seat.agent.clone();
7021
7022        let mut fix = FixRecord {
7023            agent: agent_id,
7024            addressed: Vec::new(),
7025            rejected: Vec::new(),
7026            notes: String::new(),
7027            committed: false,
7028            failed: None,
7029            duration_ms: 0,
7030            continuation: None,
7031        };
7032        let mut continuation = ContinuationRecord::not_needed();
7033        let mut final_seat = seat.clone();
7034        match out {
7035            AgentOutcome::Ok(o) => {
7036                fix.duration_ms = o.duration_ms;
7037                let parsed = verdict::extract_json::<FixReport>(&o.text);
7038                // A parsed report standing next to a command this same
7039                // reply's own CLI never confirmed the exit status of is
7040                // not a resolved answer — the identical `CommandEvidence`
7041                // `state.jobs` renders, read here instead of only on
7042                // display, per the completion judgment and the shown
7043                // record needing to agree.
7044                let incomplete_reason = match &parsed {
7045                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7046                        "the reply parsed, but it reported a command whose own CLI never \
7047                             confirmed an exit status"
7048                            .to_owned(),
7049                    ),
7050                    Ok(_) => None,
7051                    Err(e) => Some(e.to_string()),
7052                };
7053                match incomplete_reason {
7054                    None => {
7055                        let report = parsed.expect("checked Ok above");
7056                        fix.addressed = report.addressed;
7057                        fix.rejected = report.rejected;
7058                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7059                    }
7060                    Some(reason) => {
7061                        let (resumed_seat, resolved, failure, cont) = self
7062                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7063                            .await;
7064                        fix.duration_ms += cont.cumulative_wait_ms;
7065                        continuation = cont;
7066                        final_seat = resumed_seat;
7067                        match resolved {
7068                            Some(report) => {
7069                                fix.addressed = report.addressed;
7070                                fix.rejected = report.rejected;
7071                                fix.notes =
7072                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
7073                            }
7074                            None => fix.failed = failure,
7075                        }
7076                    }
7077                }
7078            }
7079            // The CLI's raw error JSON is not a fix report to parse.
7080            AgentOutcome::Dropped(o) => {
7081                fix.duration_ms = o.duration_ms;
7082                let why = o
7083                    .dropped
7084                    .as_ref()
7085                    .map(|d| d.why.as_str())
7086                    .unwrap_or("the CLI ended the stream without delivering its answer");
7087                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7088            }
7089            AgentOutcome::Quota(o) => {
7090                self.state.quota.push(QuotaLoss {
7091                    seat: final_seat.key.clone(),
7092                    node: "fix".to_owned(),
7093                    at: Timestamp::now(),
7094                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7095                });
7096                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7097            }
7098            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7099        }
7100        fix.continuation = Some(continuation);
7101        self.state.seats.insert(final_seat.key.clone(), final_seat);
7102        let coordination = self.coordinate(&winner.worktree).await?;
7103        if let Ok(r) = git::rescue_commit(
7104            &winner.worktree,
7105            &format!("magi: review round {round} fixes (uncommitted work)"),
7106        )
7107        .await
7108        {
7109            self.state.note_withheld("fix", &r.withheld);
7110        }
7111        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7112        fix.committed = after != before;
7113        // Judged by what `git` says moved against base, never by the
7114        // fixer's own `addressed`/`rejected` count — see
7115        // `ReviewRound::progressed`. Propagated with `?`, the same as the
7116        // `patch` snapshot above: swallowing this error would default
7117        // `diff_after` to empty, which almost always differs from a
7118        // non-empty `patch` and reads as "progressed" — exactly backwards
7119        // for a `git` failure the stagnation check cannot see through.
7120        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7121        drop(coordination);
7122        let progressed = diff_after != patch;
7123        let commit_note = if fix.committed {
7124            "committed"
7125        } else {
7126            "NO new commit"
7127        };
7128        let tree_note = if progressed {
7129            "changed vs base"
7130        } else {
7131            "unchanged vs base"
7132        };
7133        self.state.event(
7134            "fix",
7135            match &fix.failed {
7136                // Distinct on purpose from "0 addressed, 0 rejected": the
7137                // fixer's own diff still landed (blocking counts do keep
7138                // falling round over round), only its adoption report did
7139                // not come back, so this must never read like every
7140                // finding was reviewed and declined.
7141                Some(reason) => {
7142                    format!(
7143                        "round {round}: fixer's adoption report was lost ({reason}); \
7144                             {commit_note}, tree {tree_note}"
7145                    )
7146                }
7147                None => format!(
7148                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7149                         {tree_note}{}",
7150                    fix.addressed.len(),
7151                    fix.rejected.len(),
7152                    if continuation.outcome == ContinuationOutcome::Resumed {
7153                        format!(
7154                            " (adoption report recovered after {} continuation(s))",
7155                            continuation.attempts
7156                        )
7157                    } else {
7158                        String::new()
7159                    },
7160                ),
7161            },
7162        );
7163        round_record.fix = Some(fix);
7164        round_record.progressed = progressed;
7165        Ok(FixStep::Recorded {
7166            report_lost: matches!(
7167                continuation.outcome,
7168                ContinuationOutcome::Exhausted
7169                    | ContinuationOutcome::QuotaLost
7170                    | ContinuationOutcome::NoSession
7171            ),
7172        })
7173    }
7174
7175    /// Decide, from the last recorded round's own verification, whether
7176    /// stopping the review loop is a hand-off or a genuine block.
7177    ///
7178    /// Called once the loop has given up trying — the round budget is spent,
7179    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
7180    /// findings still open, never while a round is still clean or the
7181    /// incomplete-panel case handled inline above. Gate and e2e are facts
7182    /// about the tree; a lingering review finding is an opinion, and this
7183    /// workload's own `magi stats` puts reviewer precision low enough
7184    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
7185    /// must not by itself stand between a green, verified change and the
7186    /// human who decides what to do with it. A red e2e is not an opinion, so
7187    /// that case still blocks, with the failing command and a tail of its
7188    /// output recorded here rather than left in `run.json` for someone to go
7189    /// find.
7190    ///
7191    /// A round that deferred its own e2e (see [`Config::graph`]'s
7192    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
7193    /// only because nothing ran, and treating an empty list as a passing one
7194    /// here is exactly the "deferred painted green" bug this function exists
7195    /// to not have. When the last round's own verification never resolved —
7196    /// deferred on purpose, or a real attempt the shared build cache blocked
7197    /// — this makes (or retries) the real run, on the actual worktree this
7198    /// loop is about to stop touching, before deciding anything. A
7199    /// resource-blocked attempt is likewise never read as either green or
7200    /// red: it is evidence about the machine, not the patch (see
7201    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
7202    /// blocked cache leaves this call without deciding rather than guessing
7203    /// — the caller retries on a later reentry.
7204    /// Record, once, that the review loop handed off over a blocking finding
7205    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
7206    /// `land` asks the owner even with `land_approval` off. Called from every
7207    /// path that concludes `Gating`; a reentry keeps the first record.
7208    fn record_contested_handoff(&mut self) {
7209        if self.state.contested_handoff.is_some() {
7210            return;
7211        }
7212        let Some(contested) = self
7213            .state
7214            .reviews
7215            .last()
7216            .and_then(ReviewRound::contested_handoff)
7217        else {
7218            return;
7219        };
7220        self.state.event(
7221            "review",
7222            format!(
7223                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7224                 wait for the owner's approval",
7225                contested.findings.len(),
7226                contested.rejecters.len()
7227            ),
7228        );
7229        self.state.contested_handoff = Some(contested);
7230    }
7231
7232    async fn stop_unresolved_review(
7233        &mut self,
7234        why: &str,
7235        shell: &[String],
7236        worktree: &Path,
7237    ) -> Result<()> {
7238        self.stop_reviewing_inner(why, shell, worktree, true).await
7239    }
7240
7241    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7242        self.stop_reviewing_inner(why, shell, worktree, false).await
7243    }
7244
7245    async fn stop_reviewing_inner(
7246        &mut self,
7247        why: &str,
7248        shell: &[String],
7249        worktree: &Path,
7250        repairable: bool,
7251    ) -> Result<()> {
7252        let round_idx = self.state.reviews.len() - 1;
7253        // A deferred round and a resource-blocked one are the same shape
7254        // here: neither has a real result yet, and both get one more
7255        // attempt. Read off `e2e_status` — the single source for this —
7256        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
7257        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
7258        // still retries instead of being read as a settled result the
7259        // instant it stops being empty.
7260        let needs_catchup_run = matches!(
7261            self.state.reviews[round_idx].e2e_status(),
7262            E2eStatus::Deferred | E2eStatus::ResourceBlocked
7263        );
7264        if needs_catchup_run {
7265            let round = self.state.reviews[round_idx].round;
7266            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7267            let commands = self.state.config.verify.e2e.clone();
7268            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7269            let cache_dir = self.state.config.cache_dir();
7270            let context = format!(
7271                "round {round}: verification unresolved, catching up before the final decision"
7272            );
7273            let wait = self.repository_wait();
7274            let (outcomes, verify_retried) = with_cache_lease(
7275                &mut self.state,
7276                cache_dir.as_deref(),
7277                "e2e",
7278                "e2e",
7279                worktree,
7280                &attempted_head,
7281                timeout,
7282                &context,
7283                &wait,
7284                |state, budget| {
7285                    let shell = shell.to_vec();
7286                    let commands = commands.clone();
7287                    let context = context.clone();
7288                    async move {
7289                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7290                            .await
7291                    }
7292                },
7293            )
7294            .await?;
7295            let last = &mut self.state.reviews[round_idx];
7296            last.e2e = outcomes;
7297            last.verify_retried = verify_retried;
7298            // Always the commit and time this attempt actually targeted,
7299            // whether or not it happens to equal the reviewed `head` and
7300            // whether or not a command finished — see
7301            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
7302            // attempt is recorded too, so a later reader sees "attempted
7303            // again at T2" rather than silence.
7304            last.verified_head = Some(attempted_head);
7305            last.verified_at = Some(Timestamp::now());
7306            if verify_inconclusive(&last.e2e) {
7307                // Still not a real result: `e2e_deferred` is left exactly
7308                // as it was, so `needs_catchup_run` above reads
7309                // `ResourceBlocked` (via `e2e_status`, which checks
7310                // `resource_blocked` before `e2e_deferred`) and retries
7311                // again on the next reentry, rather than recording
7312                // contention as a red e2e and blocking the run on it.
7313                self.state.save()?;
7314                return Ok(());
7315            }
7316            last.e2e_deferred = false;
7317        }
7318        let last = &self.state.reviews[round_idx];
7319        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7320
7321        match last.e2e_status() {
7322            E2eStatus::Failed => {
7323                let red: Vec<String> = last
7324                    .e2e
7325                    .iter()
7326                    .filter(|o| !o.ok())
7327                    .map(|o| {
7328                        format!(
7329                            "`{}` -> {:?}\n{}",
7330                            o.command,
7331                            o.code,
7332                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7333                        )
7334                    })
7335                    .collect();
7336                let can_repair = last.blocking == 0 && last.fix.is_none();
7337                self.state.reviews[round_idx].clean = false;
7338                self.state
7339                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7340                self.state.status = RunStatus::Blocked;
7341                if repairable
7342                    && can_repair
7343                    && self.state.reviews.len() < self.state.config.graph.review_rounds
7344                {
7345                    self.state.status = RunStatus::Reviewing;
7346                }
7347            }
7348            // `needs_catchup_run` above already retried once this call; if
7349            // it is still blocked, this is magi's own admission it could
7350            // not get a command to run, never a verdict on the patch — the
7351            // run is left exactly where a later reentry can retry again.
7352            E2eStatus::ResourceBlocked => {
7353                self.state.event(
7354                    "review",
7355                    format!(
7356                        "{why}; e2e could not run (resource unavailable); not \
7357                         deciding yet"
7358                    ),
7359                );
7360            }
7361            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7362                self.state.event(
7363                    "review",
7364                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7365                );
7366                self.record_contested_handoff();
7367                self.state.status = RunStatus::Gating;
7368            }
7369        }
7370        self.state.save()?;
7371        Ok(())
7372    }
7373
7374    // ----------------------------------------------------------------- gate
7375
7376    async fn gate(&mut self) -> Result<()> {
7377        if self.hold_base_sync().await? {
7378            return Ok(());
7379        }
7380        // Judged by the review record itself, not by `status`: a solo
7381        // candidate's `judge`/`deliberate` skip rewrites `status` on every
7382        // reentry (see `judge`), and trusting it here is exactly how a run
7383        // that exhausted its review budget got gated and merged a second
7384        // time around. `review_conclusion` recomputes the review loop's own
7385        // verdict from the round records themselves — `Gating` for a clean
7386        // round or a hand-off (see `stop_reviewing`), anything else means the
7387        // loop is still going or genuinely blocked.
7388        // A base the winner could not be replayed onto is a decision, not a
7389        // round: there is no landing tree to gate. Read as its own record for
7390        // the same reason the review verdict is.
7391        if self.state.status == RunStatus::Failed
7392            || self
7393                .state
7394                .base_sync
7395                .as_ref()
7396                .is_some_and(|s| s.conflict.is_some())
7397            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7398                != Some(RunStatus::Gating)
7399        {
7400            return Ok(());
7401        }
7402        let Some(winner) = self.state.winner().cloned() else {
7403            return Ok(());
7404        };
7405        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7406        self.state.reconcile_verification_head(&head);
7407        self.state.save()?;
7408        let wait = self.repository_wait();
7409        if !verify_current_head(&mut self.state, &winner, &wait).await? {
7410            return Ok(());
7411        }
7412        if self.state.gate_ran {
7413            // `review_loop` derives its conclusion from the clean review
7414            // record on every reentry and therefore puts a completed run back
7415            // in `Gating`. A recorded gate is a stronger, terminal fact:
7416            // retain its original command output (or lack of any, for a repo
7417            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
7418            // doc) and restore `Blocked` on a real failure rather than
7419            // pretending the command is still running or running it a second
7420            // time. `gate_ran == false` remains the only shape — unattempted,
7421            // or a resource-blocked retry — that may still need to execute a
7422            // command.
7423            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7424                self.state.status = RunStatus::Blocked;
7425                self.state.save()?;
7426            }
7427            return Ok(());
7428        }
7429        self.state.status = RunStatus::Gating;
7430        let mut outcomes = self.run_gate(&winner).await?;
7431        let mut attempted_head = self.state.gate_head.clone();
7432        loop {
7433            // A resource-blocked outcome means the gate command never actually
7434            // ran - the shared build cache could not be acquired or confirmed
7435            // fresh in time - which is evidence about the machine, not about
7436            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
7437            // Recording it as a red gate would mark a run `Blocked` on nothing
7438            // but contention magi has already logged; leaving `self.state.gate`
7439            // empty and `self.state.gate_ran` false instead keeps the shape
7440            // this function already treats as "still needs to run" (see the
7441            // early-return above), so the next call retries the command
7442            // rather than concluding anything.
7443            if verify_inconclusive(&outcomes) {
7444                self.state.save()?;
7445                return Ok(());
7446            }
7447            if outcomes.iter().all(CommandOutcome::ok) {
7448                break;
7449            }
7450            let fixes_before = self.state.gate_fixes.len();
7451            match self.gate_fix_round(&winner, &outcomes).await? {
7452                GateFix::Retry => {
7453                    outcomes = self.run_gate(&winner).await?;
7454                    attempted_head = self.state.gate_head.clone();
7455                }
7456                GateFix::Stop => {
7457                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
7458                    if self.state.gate_fixes.len() > fixes_before
7459                        && attempted_head.as_deref() != Some(current.as_str())
7460                        && verify_current_head(&mut self.state, &winner, &wait).await?
7461                    {
7462                        // A new SHA owes a gate even when a spent fixer changed only
7463                        // commit metadata. This does not grant another repair round.
7464                        outcomes = self.run_gate(&winner).await?;
7465                        attempted_head = self.state.gate_head.clone();
7466                    } else {
7467                        break;
7468                    }
7469                }
7470                GateFix::Defer => {
7471                    self.state.save()?;
7472                    return Ok(());
7473                }
7474            }
7475        }
7476        let passed = outcomes.iter().all(CommandOutcome::ok);
7477        self.state.gate = outcomes;
7478        self.state.gate_ran = true;
7479        self.state.gate_head = attempted_head;
7480        self.state.save()?;
7481        if !verify_current_head(&mut self.state, &winner, &wait).await? {
7482            return Ok(());
7483        }
7484        if !passed {
7485            self.state.status = RunStatus::Blocked;
7486            let spent = self.state.gate_fixes.len();
7487            self.state.event(
7488                "gate",
7489                if spent == 0 {
7490                    "gate failed; not merging".to_owned()
7491                } else {
7492                    format!("gate failed after {spent} gate-fix round(s); not merging")
7493                },
7494            );
7495        }
7496        self.state.save()?;
7497        Ok(())
7498    }
7499
7500    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7501    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7502    /// after review is clean and never on a candidate awaiting judging.
7503    ///
7504    /// Never fails the run: a non-zero exit or timeout is a warning and a
7505    /// recorded outcome, and the gate remains the single arbiter. Nothing
7506    /// configured means nothing happens - no event, no commit. `commit_all`
7507    /// commits any leftover change under the neutral identity and returns
7508    /// `false` when the tree is clean, so no empty commit is ever made.
7509    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7510        let commands = state.config.verify.pre_gate.clone();
7511        if commands.is_empty() {
7512            return;
7513        }
7514        let shell = state.config.shell();
7515        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7516        let (outcomes, _) = run_commands(
7517            state,
7518            "pre_gate",
7519            "pre_gate",
7520            0,
7521            &shell,
7522            &commands,
7523            &winner.worktree,
7524            timeout,
7525        )
7526        .await;
7527        for o in &outcomes {
7528            if !o.ok() {
7529                tracing::warn!(
7530                    "pre_gate `{}` failed ({:?}); the gate decides",
7531                    o.command,
7532                    o.code
7533                );
7534            }
7535            state.event(
7536                "pre_gate",
7537                format!(
7538                    "`{}` -> {}",
7539                    o.command,
7540                    if o.ok() {
7541                        "pass".to_owned()
7542                    } else {
7543                        format!(
7544                            "FAIL ({:?})\n{}",
7545                            o.code,
7546                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7547                        )
7548                    }
7549                ),
7550            );
7551        }
7552        state.pre_gate = outcomes;
7553        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7554            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7555                Ok(head) => {
7556                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7557                    state.pre_gate_commit = Some(head);
7558                }
7559                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7560            },
7561            Ok(false) => {}
7562            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7563        }
7564        if let Err(e) = state.save() {
7565            tracing::warn!("could not persist the pre_gate record: {e:#}");
7566        }
7567    }
7568
7569    /// Run `verify.gate` once against the winner's current tree, logging one
7570    /// event per command. Empty when nothing is configured.
7571    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7572        let wait = self.repository_wait();
7573        run_landing_gate(&mut self.state, winner, &wait).await
7574    }
7575
7576    /// One bounded fix round for a failing gate.
7577    ///
7578    /// The fixer is told the failure came from the gate itself, not from a
7579    /// reviewer, and is shown the failed commands, their exit codes and a tail
7580    /// of their output - whatever `[verify].gate` holds, nothing here knows
7581    /// what those commands run. Only a normal non-zero exit that printed
7582    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7583    /// command or a full disk says nothing about the code, and a fixer sent
7584    /// after it can only appease the machine. The round is judged by what git
7585    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7586    /// again before the gate does, so a fix cannot trade a green gate for a
7587    /// red e2e unnoticed.
7588    async fn gate_fix_round(
7589        &mut self,
7590        winner: &Candidate,
7591        outcomes: &[CommandOutcome],
7592    ) -> Result<GateFix> {
7593        let cap = self.state.config.graph.gate_fix_rounds;
7594        let spent = self.state.gate_fixes.len();
7595        if spent >= cap {
7596            if cap > 0 {
7597                self.state.event(
7598                    "gate",
7599                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7600                );
7601            }
7602            return Ok(GateFix::Stop);
7603        }
7604        if !gate_fixable(outcomes) {
7605            self.state.event(
7606                "gate",
7607                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7608                 command or similar); not spending a fix round on it",
7609            );
7610            return Ok(GateFix::Stop);
7611        }
7612        let min_free = self.state.config.disk.min_free_bytes;
7613        if min_free > 0 {
7614            match crate::disk::free_bytes(&winner.worktree) {
7615                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7616                Ok(free) => {
7617                    self.state.event(
7618                        "gate",
7619                        format!(
7620                            "only {free} bytes free ({min_free} required by `[disk] \
7621                             min_free_bytes`); not spending a fix round on a failure the disk \
7622                             may explain"
7623                        ),
7624                    );
7625                    return Ok(GateFix::Stop);
7626                }
7627                Err(e) => {
7628                    self.state.event(
7629                        "gate",
7630                        format!("free disk space could not be measured ({e:#}); no fix round"),
7631                    );
7632                    return Ok(GateFix::Stop);
7633                }
7634            }
7635        }
7636
7637        let attempt = spent + 1;
7638        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7639        let base = self.landing_base();
7640        let fix_prompt = prompt::gate_fix_in(
7641            &self.state.instruction,
7642            &failed,
7643            attempt,
7644            cap,
7645            &self.state.config.graph.language,
7646            &self.state.config.graph.github_language,
7647        );
7648        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7649        let sessions = self.state.config.graph.sessions;
7650        let artifacts = self.merge_repair_artifacts();
7651        self.state.event(
7652            "gate",
7653            format!("gate failed; gate-fix round {attempt} of {cap}"),
7654        );
7655        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7656        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7657        // Persist before invoking the fixer: it may commit and then be
7658        // interrupted before its result (or the following e2e) is saved.
7659        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7660            head: before.clone(),
7661            passed: false,
7662            outcomes: Vec::new(),
7663        });
7664        self.state.save()?;
7665        let (_, seat, out) = self
7666            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7667                prompt: fix_prompt.clone(),
7668                spec,
7669                seat,
7670                cwd: winner.worktree.clone(),
7671                timeout,
7672                allow_write: true,
7673                sessions,
7674                artifacts: artifacts.clone(),
7675                stem: format!("gate-fix-{attempt}"),
7676                handover: None,
7677            })
7678            .await;
7679        // The fixer was never asked: no round is spent (nothing is recorded)
7680        // and a later reentry asks the same agent again.
7681        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7682            return Ok(GateFix::Defer);
7683        }
7684        let mut record = GateFixRecord {
7685            agent: seat.agent.clone(),
7686            failed,
7687            notes: String::new(),
7688            committed: false,
7689            error: None,
7690        };
7691        match out {
7692            AgentOutcome::Ok(o) => {
7693                // A missing report is not a failed fix: the round is judged
7694                // by the tree below, and the report only carries prose.
7695                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7696                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7697                }
7698            }
7699            AgentOutcome::Dropped(_) => {
7700                record.error = Some("the CLI dropped the stream".to_owned());
7701            }
7702            AgentOutcome::Quota(o) => {
7703                self.state.quota.push(QuotaLoss {
7704                    seat: seat.key.clone(),
7705                    node: "gate-fix".to_owned(),
7706                    at: Timestamp::now(),
7707                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7708                });
7709                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7710            }
7711            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7712        }
7713        self.state.seats.insert(seat.key.clone(), seat);
7714        let coordination = self.coordinate(&winner.worktree).await?;
7715        if let Ok(r) = git::rescue_commit(
7716            &winner.worktree,
7717            &format!("magi: gate fix {attempt} (uncommitted work)"),
7718        )
7719        .await
7720        {
7721            self.state.note_withheld("gate-fix", &r.withheld);
7722        }
7723        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7724        record.committed = after != before;
7725        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7726        let note = record.error.clone();
7727        self.state.gate_fixes.push(record);
7728        self.state.save()?;
7729        if !changed {
7730            self.state.event(
7731                "gate",
7732                match note {
7733                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7734                    None => format!("gate-fix round {attempt}: the tree did not change"),
7735                },
7736            );
7737            return Ok(GateFix::Stop);
7738        }
7739        self.state.event(
7740            "gate",
7741            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7742        );
7743
7744        drop(coordination);
7745        let wait = self.repository_wait();
7746        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7747    }
7748
7749    // ---------------------------------------------------------------- merge
7750
7751    /// One read-only rewrite by the summary's author; a text the gate still
7752    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7753    /// applies on their word or on silence. `None` means the run is parked on
7754    /// that question and nothing may be pushed or opened yet.
7755    async fn guarded_pr_message(
7756        &mut self,
7757        winner: &Candidate,
7758        facts: Option<&BranchFacts>,
7759        posting: bool,
7760    ) -> Result<Option<PrMessage>> {
7761        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7762        let head = match final_head.as_deref() {
7763            Some(h) => crate::pr_outcome::FinalHead::At(h),
7764            None => crate::pr_outcome::FinalHead::Unknown,
7765        };
7766        // Only a pull request that is about to be posted pays for a summary.
7767        let summarized = if posting {
7768            Some(self.summarize_change(winner, final_head.as_deref()).await)
7769        } else {
7770            None
7771        };
7772        // A busy shared resource is not a permanent cause: publishing an
7773        // "unavailable" description would stick (a later head move never
7774        // edits the body). Stop before anything is pushed or opened; the
7775        // run is `Blocked` and a resume asks the summary again.
7776        if self.state.resource_block.is_some() {
7777            self.state.event(
7778                "pr-summary",
7779                "a shared resource was busy; nothing was published, resume to summarize again",
7780            );
7781            return Ok(None);
7782        }
7783        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7784            anyhow::bail!(
7785                "the final commit moved during public description generation; retry before publishing"
7786            );
7787        }
7788        let note = match (&summarized, &self.state.change_summary) {
7789            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7790            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7791            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7792            (None, _) => ChangeNote::NotAsked,
7793        };
7794        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7795        if !posting {
7796            let identity = crate::scrub::Identity::current();
7797            return Ok(Some(PrMessage {
7798                title: crate::scrub::scrub(&pr.title, &identity),
7799                body: crate::scrub::scrub(&pr.body, &identity),
7800            }));
7801        }
7802        // This very text was already rejected and asked about: no second
7803        // judge call, rewrite or question, just carry the record on.
7804        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7805        if let Some(known) = self
7806            .state
7807            .github_text
7808            .clone()
7809            .filter(|g| g.fingerprint == raw_fp)
7810        {
7811            return self.settle_withheld(winner, &pr, known).await;
7812        }
7813        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7814        let mut violations =
7815            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7816        let identity = crate::scrub::Identity::current();
7817        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7818            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7819            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7820        {
7821            violations.push(crate::github_text::Violation::SensitiveData);
7822        }
7823        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7824        // only a language violation starts one; sensitive data alone goes
7825        // straight to the owner, with the redacted text as the fallback.
7826        let has_language = violations
7827            .iter()
7828            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7829        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7830            self.state.event(
7831                "github-text",
7832                if has_language {
7833                    format!("description rejected: {violations:?}; requesting one rewrite")
7834                } else {
7835                    format!("description rejected: {violations:?}; asking the owner")
7836                },
7837            );
7838            let mut rewritten = false;
7839            let mut rejected_rewrite = None;
7840            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7841            {
7842                let output = text;
7843                if let Some(title) = summary_title(&output) {
7844                    // Judge the redacted candidate that will actually be posted.
7845                    // Sensitive data alone must not discard a useful rewrite.
7846                    let raw_body = summary_without_title(&output);
7847                    let clean_title = crate::scrub::scrub(&title, &identity);
7848                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7849                    let redacted = clean_title != title || body != raw_body;
7850                    let title = clean_title;
7851                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7852                    if !complete {
7853                        self.state.event(
7854                            "github-text",
7855                            "description rewrite omitted required findings",
7856                        );
7857                    }
7858                    if !body.trim().is_empty() {
7859                        body.push_str("\n\n");
7860                        body.push_str(&public_pr_evidence(&self.state, head));
7861                    }
7862                    let marker = format!("magi:run/{}", self.state.id);
7863                    if !body.trim().is_empty() && !body.contains(&marker) {
7864                        body.push_str(&format!("\n\n{marker}"));
7865                    }
7866                    let rewrite = PrMessage {
7867                        title: title.clone(),
7868                        body: body.clone(),
7869                    };
7870                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7871                    let mut failed = crate::github_text::check_for(
7872                        &self.state.config.graph,
7873                        &title,
7874                        &body,
7875                        redecision,
7876                    );
7877                    if (!crate::github_text::shareable(&title)
7878                        || !crate::github_text::shareable(&body))
7879                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7880                    {
7881                        failed.push(crate::github_text::Violation::SensitiveData);
7882                    }
7883                    if complete
7884                        && !title.trim().is_empty()
7885                        && !body.trim().is_empty()
7886                        && !failed.is_empty()
7887                    {
7888                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7889                            title: title.clone(),
7890                            body: body.clone(),
7891                            failed: failed
7892                                .iter()
7893                                .map(|v| crate::github_text::category(*v).to_owned())
7894                                .collect(),
7895                        });
7896                    }
7897                    if complete
7898                        && !title.trim().is_empty()
7899                        && !raw_body.trim().is_empty()
7900                        && failed.is_empty()
7901                    {
7902                        decision = redecision;
7903                        pr = PrMessage { title, body };
7904                        rewritten = true;
7905                        self.state
7906                            .event("github-text", "description rewrite passed");
7907                        if redacted {
7908                            self.state.event(
7909                                "github-text",
7910                                "sensitive data removed from description rewrite",
7911                            );
7912                        }
7913                    }
7914                }
7915            }
7916            if !rewritten {
7917                self.state.event(
7918                    "github-text",
7919                    "description rewrite unavailable or rejected; asking the owner",
7920                );
7921                if let Some(w) =
7922                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7923                {
7924                    // A different text than the one asked about before: that
7925                    // question no longer describes anything.
7926                    self.retire_github_text_question("the withheld text changed");
7927                    let record = crate::run::GithubTextAsk {
7928                        rewrite: rejected_rewrite,
7929                        fingerprint: raw_fp,
7930                        title: w.title,
7931                        body: w.body,
7932                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7933                        question: None,
7934                        asks: 0,
7935                        resolved: false,
7936                        chosen_title: None,
7937                        choice: None,
7938                    };
7939                    return self.settle_withheld(winner, &pr, record).await;
7940                }
7941            }
7942        }
7943        let (title, body) =
7944            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7945        Ok(Some(PrMessage { title, body }))
7946    }
7947
7948    /// Describe the whole change from its final diff in either run mode.
7949    ///
7950    /// The summary is tied to the final commit and to the merge base of the
7951    /// resolved remote base; a record for any other pair is dropped and the
7952    /// agent asked again, so a description never outlives the diff it was
7953    /// written from. The seat is read-only and the prompt tells it to read the
7954    /// diff and the code around it. `[roles] synthesizer` selects the agents
7955    /// (unset: the default pick), each asked at most once, advancing on an
7956    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7957    /// carries the reason it is unavailable; nothing is invented then.
7958    async fn summarize_change(
7959        &mut self,
7960        winner: &Candidate,
7961        head: Option<&str>,
7962    ) -> std::result::Result<(), String> {
7963        let Some(head) = head else {
7964            return Err("the final commit could not be read".to_owned());
7965        };
7966        let remote = self.state.config.merge.remote.clone();
7967        let tracking = format!("{remote}/{}", self.state.base_branch);
7968        if !git::rev_exists(&winner.worktree, &tracking).await {
7969            return Err(format!("the remote base `{tracking}` could not be read"));
7970        }
7971        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7972            Ok(b) if !b.is_empty() => b,
7973            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7974        };
7975        let evidence = crate::pr_outcome::summary_evidence(
7976            &self.state,
7977            crate::pr_outcome::FinalHead::At(head),
7978        );
7979        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7980        if self
7981            .state
7982            .change_summary
7983            .as_ref()
7984            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7985        {
7986            return Ok(());
7987        }
7988        if self.state.change_summary.take().is_some() {
7989            self.state.event(
7990                "pr-summary",
7991                "the final diff or evidence changed; the earlier change summary was discarded",
7992            );
7993        }
7994        // Save the source snapshot even if no generation agent is available.
7995        if let Err(e) = write_artifact(
7996            &self.state,
7997            &format!("public-description-{fingerprint}.md"),
7998            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7999        ) {
8000            tracing::warn!("could not save public description evidence: {e:#}");
8001        }
8002        let chain = match agent::pick_chain(
8003            &self.state.config.agents,
8004            self.state.config.roles.synthesizer.as_ref(),
8005            &agent::installed,
8006            "synthesizer",
8007        ) {
8008            Ok(chain) => chain,
8009            Err(e) => {
8010                tracing::warn!("no agent for the change summary: {e:#}");
8011                self.state.event("pr-summary", "no summary agent available");
8012                return Err("no agent was available to summarize it".to_owned());
8013            }
8014        };
8015        let subjects = git::subjects(&winner.worktree, &base, head)
8016            .await
8017            .unwrap_or_default();
8018        let stat = git::diff_stat(&winner.worktree, &base, head)
8019            .await
8020            .unwrap_or_default();
8021        let stat: String = stat.chars().take(6_000).collect();
8022        let mut prompt = crate::prompt::change_summary_in(
8023            &base,
8024            head,
8025            &subjects,
8026            stat.trim(),
8027            &self.state.config.graph.github_language,
8028        );
8029        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8030        let sessions = self.state.config.graph.sessions;
8031        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8032        let prompts = self.state.config.prompts.clone();
8033        let cache = self.state.config.cache_dir();
8034        let run = self.state.id.clone();
8035        for (i, spec) in chain.into_iter().enumerate() {
8036            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8037            let github_spec = match agent::for_github_text(
8038                &spec,
8039                &self.state.config.graph.github_language,
8040                &winner.worktree,
8041            ) {
8042                Ok(spec) => spec,
8043                Err(e) => {
8044                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8045                    continue;
8046                }
8047            };
8048            let job = SeatJob {
8049                spec: github_spec,
8050                seat,
8051                cwd: winner.worktree.clone(),
8052                prompt: prompt.clone(),
8053                timeout,
8054                allow_write: false,
8055                sessions,
8056                artifacts: agent::artifacts_dir(&self.state.dir()),
8057                stem: if i == 0 {
8058                    "change-summary".to_owned()
8059                } else {
8060                    format!("change-summary-{}", spec.id)
8061                },
8062                handover: None,
8063            };
8064            let ctx = WaveCtx {
8065                run: &run,
8066                node: "pr-summary",
8067                prompts: &prompts,
8068                cache: cache.as_deref(),
8069                round: None,
8070                carry_seats: false,
8071                pauses: Some((&self.pause, &self.interrupt)),
8072            };
8073            let (seat, outcome) =
8074                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8075            self.state.seats.insert(seat.key.clone(), seat);
8076            let text = match outcome {
8077                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8078                AgentOutcome::ResourceBlocked(why) => {
8079                    // No agent was asked: the run stops resumably before
8080                    // anything is published (see `guarded_pr_message`).
8081                    if self.state.resource_block.is_none() {
8082                        self.state.resource_block = Some(crate::run::ResourceBlock {
8083                            cause: crate::run::ResourceBlockCause::Other,
8084                            node: "pr-summary".to_owned(),
8085                            kind: "shared resource".to_owned(),
8086                            detail: why,
8087                            at: Timestamp::now(),
8088                        });
8089                    }
8090                    return Err("a shared resource was busy".to_owned());
8091                }
8092                _ => continue,
8093            };
8094            let blind = &self.state.config.blind;
8095            let Some(title) = summary_title(&text) else {
8096                continue;
8097            };
8098            let body: String = summary_without_title(&text)
8099                .lines()
8100                .filter(|l| !l.contains("magi:run/"))
8101                .collect::<Vec<_>>()
8102                .join("\n");
8103            let body = blind::sanitize_prose(body.trim(), blind);
8104            let title = blind::sanitize_prose(&title, blind);
8105            if body.trim().is_empty() || title.trim().is_empty() {
8106                continue;
8107            }
8108            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8109                continue;
8110            }
8111            // The branch must still be the one the agent was shown.
8112            if git::rev_parse(&winner.worktree, "HEAD")
8113                .await
8114                .ok()
8115                .as_deref()
8116                != Some(head)
8117            {
8118                return Err("the branch moved while the summary was written".to_owned());
8119            }
8120            if git::merge_base(&winner.worktree, &tracking, head)
8121                .await
8122                .ok()
8123                .as_deref()
8124                != Some(base.as_str())
8125                || crate::pr_outcome::summary_evidence(
8126                    &self.state,
8127                    crate::pr_outcome::FinalHead::At(head),
8128                ) != evidence
8129            {
8130                return Err("the summary evidence changed while it was written".to_owned());
8131            }
8132            self.state.change_summary = Some(crate::run::ChangeSummary {
8133                head: head.to_owned(),
8134                base,
8135                evidence: fingerprint,
8136                title,
8137                body,
8138            });
8139            self.state.event(
8140                "pr-summary",
8141                format!("change summary written by {} for the final diff", spec.id),
8142            );
8143            if let Err(e) = self.state.save() {
8144                tracing::warn!("could not save the change summary: {e:#}");
8145            }
8146            return Ok(());
8147        }
8148        self.state
8149            .event("pr-summary", "no agent produced a change summary");
8150        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8151    }
8152
8153    /// One read-only rewrite request for a withheld description. The
8154    /// implementer's own conversation is tried first when it exists and can be
8155    /// resumed; then the `[roles] language_judge` chain (or the default pick)
8156    /// writes it in a separate conversation, skipping the implementer's agent.
8157    /// The prompt carries the whole text, so no agent needs any memory. Each
8158    /// agent is asked at most once and the chain advances on an error, quota,
8159    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
8160    /// that has run out of quota must not end the rewrite. Nothing is recorded
8161    /// as a handover.
8162    async fn rewrite_pr_text(
8163        &mut self,
8164        winner: &Candidate,
8165        pr: &PrMessage,
8166        violations: &[crate::github_text::Violation],
8167    ) -> Option<String> {
8168        let prompt = format!(
8169            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8170            pr.title, pr.body
8171        );
8172        let sessions = self.state.config.graph.sessions;
8173        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8174        // (spec, the resumed implementer's seat). A fallback seat is created
8175        // only when its turn comes, so an unused fallback leaves no record.
8176        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8177        let key = format!("impl-{}", winner.label);
8178        if let Some(spec) = self
8179            .state
8180            .config
8181            .agents
8182            .iter()
8183            .find(|a| a.id == winner.agent)
8184            .cloned()
8185            && let Some(existing) = self.state.seats.get(&key)
8186            && existing.agent == spec.id
8187            && agent::has_session(spec.kind, existing, sessions)
8188        {
8189            attempts.push((spec, Some(existing.clone())));
8190        }
8191        match agent::pick_chain(
8192            &self.state.config.agents,
8193            self.state.config.roles.language_judge.as_ref(),
8194            &agent::installed,
8195            "language_judge",
8196        ) {
8197            Ok(chain) => {
8198                for spec in chain {
8199                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8200                        continue;
8201                    }
8202                    attempts.push((spec, None));
8203                }
8204            }
8205            Err(e) if attempts.is_empty() => {
8206                tracing::warn!("no agent for the description rewrite: {e:#}");
8207                self.state
8208                    .event("github-text", "no rewrite agent available");
8209                return None;
8210            }
8211            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8212        }
8213        let prompts = self.state.config.prompts.clone();
8214        let cache = self.state.config.cache_dir();
8215        let run = self.state.id.clone();
8216        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8217            let resumed = resumed_seat.is_some();
8218            let seat = resumed_seat.unwrap_or_else(|| {
8219                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8220            });
8221            let github_spec = match agent::for_github_text(
8222                &spec,
8223                &self.state.config.graph.github_language,
8224                &winner.worktree,
8225            ) {
8226                Ok(spec) => spec,
8227                Err(e) => {
8228                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8229                    continue;
8230                }
8231            };
8232            let job = SeatJob {
8233                spec: github_spec,
8234                seat,
8235                cwd: winner.worktree.clone(),
8236                prompt: prompt.clone(),
8237                timeout: retry_budget(timeout_full, resumed),
8238                allow_write: false,
8239                sessions,
8240                artifacts: agent::artifacts_dir(&self.state.dir()),
8241                stem: if i == 0 {
8242                    "github-text-rewrite".to_owned()
8243                } else {
8244                    format!("github-text-rewrite-{}", spec.id)
8245                },
8246                handover: None,
8247            };
8248            let ctx = WaveCtx {
8249                run: &run,
8250                node: "github-text",
8251                prompts: &prompts,
8252                cache: cache.as_deref(),
8253                round: None,
8254                carry_seats: false,
8255                pauses: Some((&self.pause, &self.interrupt)),
8256            };
8257            let (seat, outcome) =
8258                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8259            self.state.seats.insert(seat.key.clone(), seat);
8260            match outcome {
8261                AgentOutcome::Ok(o)
8262                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8263                {
8264                    return Some(o.text);
8265                }
8266                AgentOutcome::ResourceBlocked(_) => return None,
8267                _ => {}
8268            }
8269        }
8270        None
8271    }
8272
8273    /// Abandon the standing posting-gate question, if any.
8274    fn retire_github_text_question(&mut self, why: &str) {
8275        let Some(id) = self
8276            .state
8277            .github_text
8278            .as_ref()
8279            .and_then(|g| g.question.clone())
8280        else {
8281            return;
8282        };
8283        let _ = ask::Questions::open().update(&id, |q| {
8284            q.abandon(why);
8285            Ok(())
8286        });
8287    }
8288
8289    /// Carry a withheld text to its end: file the question, park while it is
8290    /// open, and on an answer, silence or timeout pick the text to post. The
8291    /// decision is saved before it is used, so it applies once.
8292    async fn settle_withheld(
8293        &mut self,
8294        winner: &Candidate,
8295        pr: &PrMessage,
8296        mut g: crate::run::GithubTextAsk,
8297    ) -> Result<Option<PrMessage>> {
8298        use crate::github_text as gt;
8299        let store = ask::Questions::open();
8300        let timeout = self.state.config.graph.answer_timeout;
8301        if !g.resolved {
8302            let mut standing = g
8303                .question
8304                .as_ref()
8305                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8306            if let Some(q) = standing.as_ref()
8307                && q.status.open()
8308                && gt::expired(q, timeout)
8309            {
8310                let why = format!("no answer within {}s of asking", timeout.max(1));
8311                standing = store
8312                    .update(&q.id, |q| {
8313                        q.abandon(&why);
8314                        Ok(())
8315                    })
8316                    .ok()
8317                    .map(|(q, ())| q);
8318            }
8319            let mut ask_again: Option<Option<Vec<String>>> = None;
8320            match standing {
8321                // Recorded but never written (a stop between the two saves),
8322                // or removed: ask again without spending the retry.
8323                None if g.asks <= 1 => {
8324                    g.asks = 0;
8325                    ask_again = Some(None);
8326                }
8327                None => {
8328                    g.resolved = true;
8329                    g.chosen_title = None;
8330                    g.choice = Some(GithubTextChoice::Fallback);
8331                }
8332                Some(q) if q.status.open() => {
8333                    self.park_on_github_text(&g)?;
8334                    return Ok(None);
8335                }
8336                Some(q) => match gt::read_reply(&q) {
8337                    gt::Reply::Fallback => {
8338                        g.resolved = true;
8339                        g.chosen_title = None;
8340                        g.choice = Some(GithubTextChoice::Fallback);
8341                    }
8342                    gt::Reply::Original => {
8343                        g.resolved = true;
8344                        g.chosen_title = None;
8345                        g.choice = Some(GithubTextChoice::Original);
8346                    }
8347                    gt::Reply::Rewrite => {
8348                        g.resolved = true;
8349                        g.chosen_title = None;
8350                        g.choice = Some(if g.rewrite.is_some() {
8351                            GithubTextChoice::Rewrite
8352                        } else {
8353                            GithubTextChoice::Fallback
8354                        });
8355                    }
8356                    gt::Reply::Title(text) => {
8357                        let first = text
8358                            .lines()
8359                            .map(str::trim)
8360                            .find(|l| !l.is_empty())
8361                            .unwrap_or("");
8362                        let decision = if first.is_empty() {
8363                            None
8364                        } else {
8365                            let candidate = PrMessage {
8366                                title: first.to_owned(),
8367                                body: String::new(),
8368                            };
8369                            self.judge_pr_language(&winner.worktree, &candidate).await
8370                        };
8371                        match gt::vet_title(
8372                            &text,
8373                            decision,
8374                            self.state.config.graph.github_english(),
8375                        ) {
8376                            Ok(title) => {
8377                                g.resolved = true;
8378                                g.chosen_title = Some(title);
8379                                g.choice = Some(GithubTextChoice::MyTitle);
8380                            }
8381                            Err(cats) if g.asks < 2 => {
8382                                ask_again = Some(Some(
8383                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8384                                ))
8385                            }
8386                            Err(_) => {
8387                                g.resolved = true;
8388                                g.chosen_title = None;
8389                                g.choice = Some(GithubTextChoice::Fallback);
8390                            }
8391                        }
8392                    }
8393                },
8394            }
8395            if let Some(rejected) = ask_again {
8396                // A retry (after a rejected replacement title) is still about
8397                // the original text: the same ORIGINAL / FALLBACK comparison.
8398                let w = gt::Withheld {
8399                    title: g.title,
8400                    body: g.body,
8401                    categories: g.categories.clone(),
8402                };
8403                // Named by the original text's fingerprint alone, so a resume
8404                // rewrites the same file and the detail stays identical.
8405                let artifact = match crate::run::write_artifact(
8406                    &self.state,
8407                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
8408                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8409                ) {
8410                    Ok(p) => Some(p),
8411                    Err(e) => {
8412                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
8413                        None
8414                    }
8415                };
8416                let (detail, comparison) = gt::question_content(
8417                    &w,
8418                    &pr.title,
8419                    &pr.body,
8420                    &self.state.id,
8421                    rejected.as_deref(),
8422                    artifact.as_deref(),
8423                    g.rewrite.as_ref(),
8424                );
8425                let mut q = ask::Question::new(
8426                    self.state.id.clone(),
8427                    gt::ASK_NODE.to_owned(),
8428                    gt::ASK_SEAT.to_owned(),
8429                    gt::question_summary(&self.state.config.graph.language, &w),
8430                    detail,
8431                    gt::question_choices(&w, g.rewrite.as_ref()),
8432                );
8433                q.posting_comparison = Some(comparison);
8434                q.answer_timeout = timeout;
8435                // Recorded before it is written: a stop in between leaves a
8436                // record naming a question the resume then files again.
8437                g.question = Some(q.id.clone());
8438                g.asks += 1;
8439                self.state.github_text = Some(g.clone());
8440                self.state.event(
8441                    "github-text",
8442                    format!("asking the owner about the withheld text ({})", q.short()),
8443                );
8444                self.state.save()?;
8445                store
8446                    .put_executable(&mut q)
8447                    .context("file the posting-gate question")?;
8448                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8449                    tracing::warn!(
8450                        "could not notify about posting-gate question {}: {e:#}",
8451                        q.short()
8452                    );
8453                }
8454                self.park_on_github_text(&g)?;
8455                return Ok(None);
8456            }
8457        }
8458        let resolved = g.chosen_title.clone();
8459        let original = g.choice == Some(GithubTextChoice::Original);
8460        // Field names only: the event log never records the text.
8461        self.state.event(
8462            "github-text",
8463            if original {
8464                format!(
8465                    "posting the original text for: {}",
8466                    match (g.title, g.body) {
8467                        (true, true) => "title, description",
8468                        (true, false) => "title",
8469                        _ => "description",
8470                    }
8471                )
8472            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8473                "posting the saved rewrite title and description".to_owned()
8474            } else if resolved.is_some() {
8475                "posting the owner's replacement title".to_owned()
8476            } else {
8477                "posting the neutral text for the withheld fields".to_owned()
8478            },
8479        );
8480        self.state.github_text = Some(g.clone());
8481        self.state.save()?;
8482        if g.choice == Some(GithubTextChoice::Rewrite)
8483            && let Some(r) = g.rewrite.as_ref()
8484        {
8485            return Ok(Some(PrMessage {
8486                title: r.title.clone(),
8487                body: r.body.clone(),
8488            }));
8489        }
8490        let w = gt::Withheld {
8491            title: g.title,
8492            body: g.body,
8493            categories: g.categories.clone(),
8494        };
8495        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8496        if let Some(t) = resolved {
8497            title = t;
8498        }
8499        // Every withheld field is now neutral or vetted English.
8500        let english = gt::LanguageDecision {
8501            title_english: true,
8502            body_english: true,
8503            body_complete: true,
8504        };
8505        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8506        if original {
8507            // The owner's explicit choice overrides the gate for the fields it
8508            // was made for; the others keep what the gate produced.
8509            if g.title {
8510                title = pr.title.clone();
8511            }
8512            if g.body {
8513                body = pr.body.clone();
8514            }
8515        }
8516        Ok(Some(PrMessage { title, body }))
8517    }
8518
8519    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8520        self.state.github_text = Some(g.clone());
8521        self.state.end_pass_parked(ParkCause::Question);
8522        self.state.parked = true;
8523        self.state.event(
8524            "github-text",
8525            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8526        );
8527        self.state.save()
8528    }
8529
8530    /// Ask `[roles] language_judge` about this text and record which source
8531    /// decided; the text itself is never recorded.
8532    async fn judge_pr_language(
8533        &mut self,
8534        cwd: &Path,
8535        pr: &PrMessage,
8536    ) -> Option<crate::github_text::LanguageDecision> {
8537        self.state.config.roles.language_judge.as_ref()?;
8538        if !self.state.config.graph.language_gate() {
8539            return None;
8540        }
8541        let decision =
8542            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8543        self.state.event(
8544            "github-text",
8545            if decision.is_some() {
8546                "language decided by the language judge"
8547            } else {
8548                "language judge unavailable; using built-in heuristics"
8549            },
8550        );
8551        decision
8552    }
8553
8554    async fn merge(&mut self) -> Result<()> {
8555        if self.resource_halt()? || self.hold_base_sync().await? {
8556            return Ok(());
8557        }
8558        // Same reasoning as `gate`: ask the review and gate records directly
8559        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8560        // skip can rewrite on reentry to something that no longer says
8561        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8562        // so a hand-off (open findings, green verification) reaches merge
8563        // exactly like a genuinely clean round does.
8564        //
8565        // A run resumed mid-`land` never reaches here at all: `execute`
8566        // recognises `RunStatus::Landing` before it even calls `prep`, and
8567        // routes straight to `run_land` instead. That has to happen a level
8568        // up from this function, not with a check in here, because
8569        // `review_loop`'s own status recomputation (see its doc) runs
8570        // *before* `merge` on every reentry and would otherwise overwrite
8571        // the `Landing` marker with `Gating` before this node ever saw it.
8572        if self
8573            .state
8574            .base_sync
8575            .as_ref()
8576            .is_some_and(|s| s.conflict.is_some())
8577            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8578                != Some(RunStatus::Gating)
8579            // `gate_ran == false` is not "passed" - `gate` leaves it false
8580            // both before it has ever run and when its last attempt was
8581            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8582            // permission to merge on nothing but the review record. Only a
8583            // gate that actually ran may proceed; an intentional local
8584            // review can also finish with verification explicitly not configured.
8585            // `RunState::gate_status` computes that distinction.
8586            || !self.state.gate_allows_progress()
8587        {
8588            return Ok(());
8589        }
8590        // This node's own record, not `status`: `status == Ready` is not
8591        // unique to the harmless `MergeMode::None` path this line was
8592        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8593        // run's PR was closed without merging — and on that run `mode` is
8594        // still `Pr`, so a reentry that fell through here would push and
8595        // open a second pull request. `self.state.merge` is set exactly once
8596        // this node (or `land`) has already produced a verdict, under every
8597        // mode, which is what "already done" actually means here.
8598        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8599            return Ok(());
8600        }
8601        let Some(winner) = self.state.winner().cloned() else {
8602            return Ok(());
8603        };
8604        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8605        if self.state.gate_head.as_deref() != Some(head.as_str()) {
8606            self.state.status = RunStatus::Blocked;
8607            self.state.event(
8608                "gate",
8609                "winner changed after gate; fresh verification required",
8610            );
8611            self.state.save()?;
8612            return Ok(());
8613        }
8614        let wait = self.repository_wait();
8615        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8616            return Ok(());
8617        }
8618        let repo = self.state.repo.clone();
8619        let base = self.state.base_branch.clone();
8620        let mode = self.state.config.merge.mode;
8621        let style = self.state.config.merge.style;
8622        let facts = if is_review_run(&self.state) {
8623            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8624            let start = review_base(
8625                &repo,
8626                &self.state.config.merge.remote,
8627                &base,
8628                &self.state.base_commit,
8629                &winner.branch,
8630            )
8631            .await;
8632            branch_facts(&repo, &start, &winner.branch).await
8633        } else {
8634            None
8635        };
8636        // `None` when the base could not be freshly read: then an adopted
8637        // pull request's title is left alone.
8638        let leaked = if is_review_run(&self.state) {
8639            leaked_subjects(&self.state, &winner.branch).await
8640        } else {
8641            Some(Vec::new())
8642        };
8643        // `None`: parked on the owner's word about a withheld title or body.
8644        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8645        // the resume arrives here again.
8646        let Some(pr) = self
8647            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8648            .await?
8649        else {
8650            return Ok(());
8651        };
8652        let message = pr.commit_message();
8653
8654        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
8655        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
8656            return Ok(());
8657        }
8658        let outcome = match mode {
8659            MergeMode::None => MergeOutcome {
8660                mode,
8661                ok: true,
8662                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8663                empty: false,
8664            },
8665            MergeMode::Pr | MergeMode::Local
8666                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8667            {
8668                MergeOutcome {
8669                    mode,
8670                    ok: false,
8671                    detail: empty_candidate_detail(&self.state, &base),
8672                    empty: true,
8673                }
8674            }
8675            MergeMode::Local => {
8676                let on = git::current_branch(&repo).await?;
8677                if on.as_deref() != Some(base.as_str()) {
8678                    MergeOutcome {
8679                        mode,
8680                        ok: false,
8681                        detail: format!(
8682                            "{} has {} checked out, not the base branch {base}",
8683                            repo.display(),
8684                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8685                        ),
8686                        empty: false,
8687                    }
8688                } else if !git::is_clean(&repo).await? {
8689                    MergeOutcome {
8690                        mode,
8691                        ok: false,
8692                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8693                        empty: false,
8694                    }
8695                } else {
8696                    let out = match style {
8697                        MergeStyle::Merge => {
8698                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8699                        }
8700                        MergeStyle::Squash => {
8701                            git::merge_squash(&repo, &winner.branch, &message).await?
8702                        }
8703                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8704                    };
8705                    MergeOutcome {
8706                        mode,
8707                        ok: out.ok(),
8708                        detail: if out.ok() { out.stdout } else { out.stderr },
8709                        empty: false,
8710                    }
8711                }
8712            }
8713            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8714                self.state.pr_publication_pending = true;
8715                self.state.save()?;
8716                let out =
8717                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8718                self.state.pr_publication_pending = out.is_err();
8719                MergeOutcome {
8720                    mode,
8721                    ok: out.is_ok(),
8722                    detail: match out {
8723                        Ok(url) => url,
8724                        Err(e) => format!("{e:#}"),
8725                    },
8726                    empty: false,
8727                }
8728            }
8729            MergeMode::Pr => {
8730                let remote = self.state.config.merge.remote.clone();
8731                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8732                let pushed = if self.state.foreign_checkout.is_some() {
8733                    let refspec = format!("{delivery_head}:refs/heads/{public}");
8734                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
8735                } else {
8736                    git::push(&winner.worktree, &remote, &winner.branch).await?
8737                };
8738                if !pushed.ok() {
8739                    MergeOutcome {
8740                        mode,
8741                        ok: false,
8742                        detail: pushed.stderr,
8743                        empty: false,
8744                    }
8745                } else {
8746                    if self.state.foreign_checkout.is_some() {
8747                        self.state.pr_pushed_head = Some(delivery_head.clone());
8748                    }
8749                    // A retry or resume of a run whose branch already has an
8750                    // open pull request adopts it rather than failing on a
8751                    // duplicate. Only this winner branch into this base:
8752                    // `branch_for` derives the name from the run id, so a
8753                    // different run's pull request never matches.
8754                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
8755                    let out = match pr_merge_plan(found) {
8756                        PrPlan::Create => {
8757                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
8758                                .await
8759                        }
8760                        PrPlan::Adopt { url, title } => {
8761                            self.state
8762                                .event("merge", format!("Pr: adopted open pull request {url}"));
8763                            if self.state.github_text.as_ref().is_some_and(|g| {
8764                                g.resolved
8765                                    && g.choice == Some(GithubTextChoice::Rewrite)
8766                                    && g.rewrite.is_some()
8767                            }) {
8768                                // The owner chose the entire pair, even if the
8769                                // existing title already matches or was edited.
8770                                land::set_pr_text_verbatim(
8771                                    &winner.worktree,
8772                                    &url,
8773                                    &pr.title,
8774                                    &pr.body,
8775                                )
8776                                .await
8777                                .map(|()| url)
8778                            } else {
8779                                if title != pr.title
8780                                    && (!is_review_run(&self.state)
8781                                        || leaked
8782                                            .as_deref()
8783                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8784                                    && let Err(e) =
8785                                        if self.state.github_text.as_ref().is_some_and(|g| {
8786                                            g.resolved
8787                                                && g.title
8788                                                && g.choice == Some(GithubTextChoice::Original)
8789                                        }) {
8790                                            // The owner chose to post this title unmodified.
8791                                            land::set_pr_title_verbatim(
8792                                                &winner.worktree,
8793                                                &url,
8794                                                &pr.title,
8795                                            )
8796                                            .await
8797                                        } else {
8798                                            land::set_pr_title(
8799                                                &mut self.state,
8800                                                &winner.worktree,
8801                                                &url,
8802                                                &pr.title,
8803                                            )
8804                                            .await
8805                                        }
8806                                {
8807                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8808                                    self.state
8809                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8810                                }
8811                                Ok(url)
8812                            }
8813                        }
8814                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8815                    };
8816                    match out {
8817                        Ok(url) => MergeOutcome {
8818                            mode,
8819                            ok: true,
8820                            detail: url,
8821                            empty: false,
8822                        },
8823                        Err(e) => MergeOutcome {
8824                            mode,
8825                            ok: false,
8826                            detail: e.to_string(),
8827                            empty: false,
8828                        },
8829                    }
8830                }
8831            }
8832        };
8833
8834        self.state.status = match (mode, outcome.ok) {
8835            (MergeMode::None, _) => RunStatus::Ready,
8836            (_, true) if outcome.empty => RunStatus::Ready,
8837            (MergeMode::Pr, true) => RunStatus::Ready,
8838            (_, true) => RunStatus::Merged,
8839            (_, false) => RunStatus::Blocked,
8840        };
8841        self.state.event(
8842            "merge",
8843            format!(
8844                "{:?}: {}",
8845                mode,
8846                outcome.detail.lines().next().unwrap_or("")
8847            ),
8848        );
8849        self.state.merge = Some(outcome);
8850        self.state.save()?;
8851
8852        // The PR is open and the run would historically stop here, leaving the
8853        // operator to watch checks, feed review comments back to a fixer, and
8854        // merge. That was done by hand six times in one session before this
8855        // existed. Opt-in, because merging is the one irreversible thing magi
8856        // can do to a repository.
8857        if self.state.config.graph.land
8858            && mode == MergeMode::Pr
8859            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8860        {
8861            self.run_land().await?;
8862        }
8863        // `run_land` may have left `status` at `Landing` - still waiting on
8864        // CI or the owner's approval, not actually settled - so this has to
8865        // read whatever `status` ended up as here, not the `Merged` this
8866        // function set a few lines up.
8867        self.settle_questions();
8868        Ok(())
8869    }
8870
8871    /// Enter `land`.
8872    ///
8873    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8874    /// a resumed run's re-entry. `land::land` itself is what serialises the
8875    /// two git-mutating moments inside the loop — the rebase push and
8876    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8877    /// to hold a lock across the whole call, and doing so would serialise
8878    /// this run's CI wait against a *different* run's land-approval resume
8879    /// in the same repository, which is exactly the "must not wait on
8880    /// another task" property the daemon's slot-freeing exists to give.
8881    async fn run_land(&mut self) -> Result<()> {
8882        crate::pr_head::ensure_owner(&self.state)?;
8883        let url = self
8884            .state
8885            .pr
8886            .as_ref()
8887            .map(|pr| pr.url.clone())
8888            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8889            .unwrap_or_default();
8890        let url = url.lines().next().unwrap_or("").trim().to_owned();
8891        if !url.starts_with("http") {
8892            return Ok(());
8893        }
8894        // A land failure is not a lost run: the work is on a branch and the
8895        // pull request is open, which is exactly where a human takes over.
8896        loop {
8897            match land::land(&mut self.state, &url).await {
8898                Ok(_)
8899                    if self
8900                        .state
8901                        .merge_repairs
8902                        .last()
8903                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8904                {
8905                    if self.repair_merge_review().await? {
8906                        continue;
8907                    }
8908                }
8909                Ok(pr) if self.state.parked => {
8910                    // `land` already saved the parked marker; nothing here
8911                    // overrides `status` back to a terminal value while an
8912                    // approval is still outstanding.
8913                    let _ = pr;
8914                }
8915                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8916                    self.state.status = RunStatus::Superseded;
8917                    self.state.save()?;
8918                }
8919                Ok(pr) => {
8920                    self.state.status = match pr.state {
8921                        land::PrLifecycle::Merged => RunStatus::Merged,
8922                        _ => RunStatus::Blocked,
8923                    };
8924                    // Downstream of a confirmed merge only - see
8925                    // `bump::should_release_bump`'s own doc for why this one
8926                    // check covers all three of `land`'s success paths.
8927                    // Best-effort: the run already landed, so a failure here
8928                    // (the decision call, `gh`, `cargo`) is recorded and never
8929                    // turns a landed run into a failed one.
8930                    if bump::should_release_bump(self.state.status)
8931                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8932                    {
8933                        // The event is the run's own record. Not-eligible cases
8934                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8935                        // `Err` is a bump that was tried and failed:
8936                        // `after_merge` itself raises the operator notice for
8937                        // that, whether or not a release PR exists yet.
8938                        self.state
8939                            .event("bump", format!("release bump skipped: {e:#}"));
8940                    }
8941                    // Independent of the bump, and best-effort in the same way:
8942                    // findings the merge left open become follow-up tasks.
8943                    if self.state.status == RunStatus::Merged {
8944                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8945                    }
8946                    self.state.save()?;
8947                }
8948                Err(e) => {
8949                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8950                        return Err(e);
8951                    }
8952                    self.state.status = RunStatus::Blocked;
8953                    self.state.event("land", format!("gave up: {e}"));
8954                    self.state.save()?;
8955                }
8956            }
8957            break;
8958        }
8959        Ok(())
8960    }
8961
8962    /// One repair cycle stays under the existing driver/task claim. An
8963    /// interrupted fixer is rolled back and held rather than run twice.
8964    async fn repair_merge_review(&mut self) -> Result<bool> {
8965        use crate::run::MergeRepairPhase as Phase;
8966        crate::pr_head::ensure_owner(&self.state)?;
8967        let i = self.state.merge_repairs.len() - 1;
8968        let request = self.state.merge_repairs[i].clone();
8969        if request.phase == Phase::Fixing {
8970            let restored = match self.state.winner().cloned() {
8971                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8972                None => Err(anyhow::anyhow!("repair has no winning branch")),
8973            };
8974            let why = match restored {
8975                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8976                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8977                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8978            };
8979            self.hold_merge_repair(i, why)?;
8980            return Ok(false);
8981        }
8982        // A completed repair gate is a publication checkpoint. On reentry,
8983        // reject external changes before the ordinary gate path can replace
8984        // that evidence by verifying a new head. Requested repairs still
8985        // start their own review generation below.
8986        if request.phase == Phase::Reviewing && self.state.gate_ran {
8987            let winner = self
8988                .state
8989                .winner()
8990                .context("repair has no winning branch")?;
8991            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8992            if self.state.gate_head.as_deref() != Some(head.as_str())
8993                || !git::is_clean(&winner.worktree).await?
8994            {
8995                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8996                return Ok(false);
8997            }
8998        }
8999        let failure = if i >= self.state.config.graph.land_rounds
9000            || self.state.config.graph.review_rounds == 0
9001        {
9002            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9003        } else {
9004            None
9005        };
9006        if let Some(why) = failure {
9007            self.hold_merge_repair(i, why)?;
9008            return Ok(false);
9009        }
9010        if request.phase == Phase::Requested {
9011            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9012                Ok(pr) => pr,
9013                Err(e) => {
9014                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9015                    return Ok(false);
9016                }
9017            };
9018            self.state.merge_repairs[i].phase = Phase::Fixing;
9019            self.state.land_approval = None;
9020            self.state.land_armed_head = None;
9021            self.state.save()?;
9022            // Serialize the exact records, including finding ids/details,
9023            // reconsideration reasons, final votes and verification output.
9024            let evidence = serde_json::to_string_pretty(&request)?;
9025            let reason = format!(
9026                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9027            );
9028            let budget = self.state.config.graph.land_rounds;
9029            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9030            match result {
9031                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9032                Ok(land::Fixed::Failed(why)) => {
9033                    self.hold_merge_repair(
9034                        i,
9035                        format!(
9036                            "repair failed: {why}; inspect the branch and run an explicit review"
9037                        ),
9038                    )?;
9039                    return Ok(false);
9040                }
9041                Err(e) => {
9042                    self.hold_merge_repair(
9043                        i,
9044                        format!(
9045                            "repair failed: {e:#}; inspect the branch and run an explicit review"
9046                        ),
9047                    )?;
9048                    return Ok(false);
9049                }
9050            }
9051            // Even no change gets a real new reviewer wave. The previous
9052            // evidence remains in the request; a former clean/concluded round
9053            // cannot short-circuit the new review or gate.
9054            self.begin_merge_repair_review(i)?;
9055        }
9056        self.review_loop().await?;
9057        self.gate().await?;
9058        if self.resource_halt()? || self.state.parked {
9059            return Ok(false);
9060        }
9061        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9062            != Some(RunStatus::Gating)
9063            || !self.state.gate_allows_progress()
9064        {
9065            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9066            return Ok(false);
9067        }
9068        // Review and gate may have added commits after the initial repair.
9069        // Publish exactly their final head to the existing PR, with a lease.
9070        let winner = self
9071            .state
9072            .winner()
9073            .cloned()
9074            .context("repair has no winning branch")?;
9075        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9076        if self.state.gate_head.as_deref() != Some(after.as_str())
9077            || !git::is_clean(&winner.worktree).await?
9078        {
9079            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9080            return Ok(false);
9081        }
9082        let expected = self
9083            .state
9084            .pr_pushed_head
9085            .as_deref()
9086            .unwrap_or(&request.head)
9087            .to_owned();
9088        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9089        // Push the checked commit itself: a local ref moving after the check
9090        // must not change which commit the forge receives.
9091        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9092            return Ok(false);
9093        }
9094        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9095        let refspec = format!("{after}:refs/heads/{public}");
9096        let push = git::git_raw(
9097            &winner.worktree,
9098            &["push", &lease, &self.state.config.merge.remote, &refspec],
9099        )
9100        .await;
9101        match push {
9102            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9103            Ok(out) => {
9104                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9105                return Ok(false);
9106            }
9107            Err(e) => {
9108                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9109                return Ok(false);
9110            }
9111        }
9112        self.state.merge_repairs[i].phase = Phase::Complete;
9113        self.state.status = RunStatus::Landing;
9114        self.state.event(
9115            "repair",
9116            "new review and gate completed; fresh owner merge decision required",
9117        );
9118        self.state.save()?;
9119        Ok(true)
9120    }
9121
9122    fn merge_repair_artifacts(&self) -> PathBuf {
9123        let artifacts = agent::artifacts_dir(&self.state.dir());
9124        match self
9125            .state
9126            .merge_repairs
9127            .last()
9128            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9129        {
9130            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9131            None => artifacts,
9132        }
9133    }
9134
9135    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9136        if let Some(e2e) = self.state.head_e2e.take() {
9137            self.state
9138                .verification_history
9139                .push(crate::run::VerificationEvidence {
9140                    superseded_by: None,
9141                    archived_at: Some(Timestamp::now()),
9142                    fixes_spent: Some(self.state.gate_fixes.len()),
9143                    gate_head: None,
9144                    gate_ran: false,
9145                    gate: Vec::new(),
9146                    e2e: Some(e2e),
9147                });
9148        }
9149        self.state.reviews.clear();
9150        self.state.gate.clear();
9151        self.state.gate_ran = false;
9152        self.state.gate_head = None;
9153        self.state.gate_fixes.clear();
9154        self.state.gate_repair_e2e = None;
9155        self.state.pre_gate.clear();
9156        self.state.pre_gate_commit = None;
9157        self.state.contested_handoff = None;
9158        self.state.land_reruns = None;
9159        self.state.land_approval = None;
9160        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9161        self.state.status = RunStatus::Reviewing;
9162        self.state.save()
9163    }
9164
9165    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9166        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9167        self.state.merge_repairs[index].failure = Some(why.clone());
9168        self.state.status = RunStatus::Blocked;
9169        self.state.event("repair", why);
9170        self.state.save()
9171    }
9172
9173    // -------------------------------------------------------------- helpers
9174
9175    /// Fetch or create a seat, keeping its conversation across nodes.
9176    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9177        if let Some(existing) = self.state.seats.get(key)
9178            && existing.agent == agent
9179        {
9180            return existing.clone();
9181        }
9182        // A seat that changes agent mints its session id from the agent too,
9183        // like a handover: the old agent's uuid is already taken by the CLI.
9184        let fresh = if self.state.seats.contains_key(key) {
9185            handover_seat(key, agent, self.state.next_seat_seed())
9186        } else {
9187            SeatState::new(key, agent, self.state.seed)
9188        };
9189        self.state.seats.insert(key.to_owned(), fresh.clone());
9190        fresh
9191    }
9192
9193    /// The agent now holding seat `key`: `spec`, unless a handover moved the
9194    /// seat to another roster agent, in which case that agent. Nodes that
9195    /// continue a seat's conversation (deliberation, the votes, a reviewer's
9196    /// reconsideration) must keep talking to whoever answered it, not slip
9197    /// back to the agent that failed it.
9198    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9199        match self.state.seats.get(key) {
9200            Some(s) if s.agent != spec.id => {
9201                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9202            }
9203            _ => spec,
9204        }
9205    }
9206
9207    /// A candidate rendered for judging, with the leak policy applied.
9208    fn view(&self, c: &Candidate) -> CandidateView {
9209        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9210            .unwrap_or_default();
9211        let (patch, _) = blind::sanitize_patch(
9212            &format!("candidate {} patch", c.label),
9213            &raw,
9214            &self.state.config.blind,
9215        );
9216        CandidateView {
9217            label: c.label,
9218            branch: c.branch.clone(),
9219            summary: c.summary.clone(),
9220            stat: c.stat.clone(),
9221            patch,
9222        }
9223    }
9224
9225    /// The full candidate set as prompt text, for seats with no live session.
9226    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9227        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9228        prompt::judge(
9229            "(see above)",
9230            &views,
9231            self.roles.judges.len(),
9232            base_short,
9233            "en",
9234        )
9235    }
9236
9237    /// The final-vote prompt with everything a seat that has no session of its
9238    /// own needs: the candidates, the seat's own ranking and reasons, and the
9239    /// anonymised deliberation it took part in (only when there was one, so a
9240    /// handed-over seat never sees more than the seat it replaces did). The
9241    /// `Final vote` heading stays first.
9242    fn vote_prompt_full(
9243        &self,
9244        j: usize,
9245        labels: &[char],
9246        language: &str,
9247        candidates: &[Candidate],
9248        base_short: &str,
9249    ) -> String {
9250        let mut text = format!(
9251            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9252            prompt::final_vote(labels, language),
9253            self.state.instruction,
9254            self.candidate_block(candidates, base_short)
9255        );
9256        if let Some(own) = self
9257            .state
9258            .judgements
9259            .get(j)
9260            .filter(|r| !r.ranking.is_empty())
9261        {
9262            let reasons = own
9263                .reasons
9264                .iter()
9265                .map(|(k, v)| format!("- {k}: {v}"))
9266                .collect::<Vec<_>>()
9267                .join("\n");
9268            text.push_str(&format!(
9269                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9270                own.ranking.iter().collect::<String>(),
9271                if reasons.is_empty() {
9272                    ""
9273                } else {
9274                    ", because:\n"
9275                }
9276            ));
9277        }
9278        if !self.state.deliberation.is_empty() {
9279            text.push_str("\n# What was argued before this vote\n");
9280            for t in self.transcript(&[], j) {
9281                text.push_str(&format!(
9282                    "\n## {}{}\n\n{}\n",
9283                    t.who,
9284                    if t.is_self { " (you)" } else { "" },
9285                    t.body.trim()
9286                ));
9287            }
9288        }
9289        text
9290    }
9291
9292    /// Anonymised transcript for judge `self_idx`.
9293    ///
9294    /// The initial rankings are always the opening statements. Seeding them
9295    /// only when no turn had been taken yet meant every judge after the first
9296    /// argued against a single voice instead of against the actual split — the
9297    /// disagreement is the information, so it is always on the table.
9298    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9299        let mut turns = Vec::new();
9300        for j in &self.state.judgements {
9301            if j.ranking.is_empty() {
9302                continue;
9303            }
9304            let reasons = j
9305                .reasons
9306                .iter()
9307                .map(|(k, v)| format!("- {k}: {v}"))
9308                .collect::<Vec<_>>()
9309                .join("\n");
9310            turns.push(Turn {
9311                who: format!("Judge {} (opening ranking)", j.judge),
9312                is_self: j.judge == self_idx + 1,
9313                body: format!(
9314                    "Ranked {}{}{reasons}",
9315                    j.ranking.iter().collect::<String>(),
9316                    if reasons.is_empty() {
9317                        ""
9318                    } else {
9319                        ", because:\n"
9320                    }
9321                ),
9322            });
9323        }
9324        for t in self
9325            .state
9326            .deliberation
9327            .iter()
9328            .flat_map(|r| r.turns.iter())
9329            .chain(current)
9330        {
9331            turns.push(Turn {
9332                who: format!("Judge {}", t.judge),
9333                is_self: t.judge == self_idx + 1,
9334                body: t.body.clone(),
9335            });
9336        }
9337        turns
9338    }
9339}
9340
9341/// Does this seat still hold the context a follow-up prompt would rely on?
9342fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9343    agent::has_session(spec.kind, seat, sessions)
9344}
9345
9346/// The next entry in `roster` after `start`, never wrapping back to the
9347/// front, whose id is not in `tried` yet.
9348///
9349/// Starts one past `start` rather than at the front of `roster`: `start` is
9350/// the seat's own original position, and a seat whose candidate slot already
9351/// sits on the roster's second entry must fall through to the third next, not
9352/// restart at the first — which is very likely a different candidate's own
9353/// agent already. Never wraps back past `start`, for the same reason: an
9354/// entry earlier in the roster than the seat's own position is almost
9355/// certainly some *other* candidate slot's own agent, and once the tail of
9356/// the roster is exhausted there are no more untried agents for *this* seat
9357/// to fall through to — the caller's fallback chain ends there, exactly as
9358/// "no further untried agents remain in the list for that seat" asks for.
9359///
9360/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
9361/// the same id twice (an operator's `roles.implementers` typo, or a
9362/// `[[agents]]` list reused across roles) must not let
9363/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
9364/// over `roster` either finds an untried id or runs out, so this always
9365/// terminates regardless of duplicates.
9366fn next_untried_in_roster<'a>(
9367    roster: &'a [AgentSpec],
9368    start: usize,
9369    tried: &BTreeSet<String>,
9370) -> Option<&'a AgentSpec> {
9371    roster
9372        .get(start + 1..)?
9373        .iter()
9374        .find(|s| !tried.contains(&s.id))
9375}
9376
9377/// The successor for a seat, shared by all four seat kinds (implement, judge,
9378/// review, advise). `others` holds the ids that *currently* occupy the other
9379/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
9380/// sees them), so roster entries beyond the seat count act as spares: a failed
9381/// seat goes to an agent no other seat holds whenever the roster permits.
9382///
9383/// `carried` is `Some` only for the review loop's carried failure history.
9384/// Order: (1) forward from `start`, never wrapping, untried, not a carried
9385/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
9386/// whole roster: untried and not another seat's occupant; (3) the plain walk
9387/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
9388/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
9389/// a better outcome than an empty seat, and it keeps the answer to "is there
9390/// a successor at all" exactly what it was before occupants were considered,
9391/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
9392/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
9393/// carried failure is a cheaper bet than doubling an agent on the panel, and
9394/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
9395/// the same round are handled one at a time, so a seat later in the batch
9396/// sees an earlier one's new occupant but not yet its own freed agent.
9397fn pick_successor<'a>(
9398    roster: &'a [AgentSpec],
9399    start: usize,
9400    tried: &BTreeSet<String>,
9401    carried: Option<&BTreeSet<String>>,
9402    others: &BTreeSet<String>,
9403) -> Option<&'a AgentSpec> {
9404    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9405    roster
9406        .get(start + 1..)
9407        .and_then(|tail| {
9408            tail.iter()
9409                .filter(free)
9410                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9411        })
9412        .or_else(|| {
9413            carried
9414                .is_some()
9415                .then(|| roster.iter().find(free))
9416                .flatten()
9417        })
9418        .or_else(|| match carried {
9419            Some(c) => next_for_seat(roster, start, tried, c),
9420            None => next_untried_in_roster(roster, start, tried),
9421        })
9422}
9423
9424/// The next agent for a seat that carries its failure history across rounds
9425/// (the review loop). `round_tried` is this round's own bound and starts
9426/// empty every round; `carried_failed` only decides priority.
9427///
9428/// First: an id walking forward from `start`, never wrapping, that is neither
9429/// tried this round nor failed in an earlier one. Only when that is exhausted
9430/// does it rescue: the first roster id (in roster order, so this one *does*
9431/// look before `start`) not yet tried this round, which is by then a carried
9432/// failure. Each id is rescued at most once per round, so it cannot loop.
9433fn next_for_seat<'a>(
9434    roster: &'a [AgentSpec],
9435    start: usize,
9436    round_tried: &BTreeSet<String>,
9437    carried_failed: &BTreeSet<String>,
9438) -> Option<&'a AgentSpec> {
9439    roster
9440        .get(start + 1..)?
9441        .iter()
9442        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9443        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9444}
9445
9446/// Where a reviewer seat starts a round: the agent that last answered it when
9447/// it is still on the roster and not marked failed, else the spec's own agent
9448/// unless it failed, else the next roster agent that has not failed, else the
9449/// spec's own agent again (the whole roster failed). Ids no longer on the
9450/// roster are ignored. An empty roster has no handover, so the spec stands.
9451fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9452    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9453        return spec;
9454    };
9455    let ok = |id: &str| !h.failed.contains(id);
9456    if let Some(last) = h.last_ok.as_deref()
9457        && ok(last)
9458        && let Some(s) = roster.iter().find(|s| s.id == last)
9459    {
9460        return s.clone();
9461    }
9462    if ok(&spec.id) {
9463        return spec;
9464    }
9465    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9466    roster
9467        .get(from + 1..)
9468        .into_iter()
9469        .flatten()
9470        .chain(roster.iter())
9471        .find(|s| ok(&s.id))
9472        .cloned()
9473        .unwrap_or(spec)
9474}
9475
9476/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
9477/// seed so a CLI that mints its session id up front (`--session-id`) never
9478/// reuses the uuid the previous agent already opened under the same seat key.
9479pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9480    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9481}
9482
9483/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
9484/// so the classifier below and the code that builds the message cannot drift.
9485const TIMED_OUT: &str = "timed out";
9486
9487impl FailClass {
9488    /// `None` for an answer; otherwise how the turn failed.
9489    fn of(out: &AgentOutcome) -> Option<Self> {
9490        match out {
9491            AgentOutcome::Ok(_) => None,
9492            AgentOutcome::Quota(_) => Some(Self::Quota),
9493            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9494            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9495            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9496            // Not a failure of the agent. Callers check for it first; `None`
9497            // here means "no class to hand over on".
9498            AgentOutcome::ResourceBlocked(_) => None,
9499        }
9500    }
9501
9502    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
9503    fn stem_word(&self) -> &'static str {
9504        match self {
9505            Self::Quota => "quota",
9506            _ => "handover",
9507        }
9508    }
9509}
9510
9511/// The message's first line with its variable parts removed — digit runs and
9512/// path-like tokens — so "exit code 2" and "exit code 7" read
9513/// as one kind of failure.
9514fn failure_signature(msg: &str) -> String {
9515    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9516    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
9517    // the historical machine key while the process diagnosis uses readable prose.
9518    if let Some(code) = line
9519        .strip_prefix("process exited with no usable output (exit code ")
9520        .and_then(|s| s.strip_suffix(')'))
9521        .and_then(|s| s.parse::<i32>().ok())
9522    {
9523        line = format!("exited with some({code}) and no usable output");
9524    } else if matches!(
9525        line.as_str(),
9526        "process exited with no usable output (no exit code reported)"
9527            | "process exited with no usable output (exit code unavailable)"
9528    ) {
9529        line = "exited with none and no usable output".to_owned();
9530    }
9531    let mut out = Vec::new();
9532    for word in line.split_whitespace() {
9533        if word.contains('/') || word.contains('\\') {
9534            out.push("<path>".to_owned());
9535            continue;
9536        }
9537        let mut w = String::new();
9538        let mut in_digits = false;
9539        for c in word.chars() {
9540            if c.is_ascii_digit() {
9541                if !in_digits {
9542                    w.push('#');
9543                }
9544                in_digits = true;
9545            } else {
9546                in_digits = false;
9547                w.push(c);
9548            }
9549        }
9550        out.push(w);
9551    }
9552    out.join(" ").chars().take(120).collect()
9553}
9554
9555/// Whether a seat that just failed with `cur` may go to the next roster agent.
9556/// A quota or a timeout always may. Any other failure may not when the agent
9557/// before it failed the same way: an error the prompt causes would otherwise
9558/// walk the whole roster. `prev` is the class of the immediately preceding
9559/// agent's failure, so a quota or timeout in between breaks the run of
9560/// identical failures by itself.
9561fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9562    match cur {
9563        FailClass::Quota | FailClass::Timeout => true,
9564        FailClass::Other(_) => prev != Some(cur),
9565    }
9566}
9567
9568/// A short human reason for a failed outcome, for the handover record.
9569fn fail_reason(out: &AgentOutcome) -> String {
9570    match out {
9571        AgentOutcome::Ok(_) => String::new(),
9572        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9573        AgentOutcome::Dropped(o) => format!(
9574            "the CLI dropped the stream ({})",
9575            o.dropped
9576                .as_ref()
9577                .map(|d| d.why.as_str())
9578                .unwrap_or("it ended without delivering its answer")
9579        ),
9580        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9581            e.lines().next().unwrap_or("").chars().take(160).collect()
9582        }
9583    }
9584}
9585
9586/// Note one handover in the run: the structured record and, in the timeline,
9587/// the sentence a person reads. A quota keeps the wording it always had.
9588pub(crate) fn record_handover(
9589    state: &mut RunState,
9590    node: &str,
9591    seat: &str,
9592    from: &str,
9593    to: &str,
9594    class: &FailClass,
9595    reason: &str,
9596) {
9597    let message = if *class == FailClass::Quota {
9598        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9599    } else {
9600        format!("{seat}: handed over {from} -> {to} ({reason})")
9601    };
9602    state.event(node, message);
9603    state.handovers.push(Handover {
9604        at: Timestamp::now(),
9605        node: node.to_owned(),
9606        seat: seat.to_owned(),
9607        from: from.to_owned(),
9608        to: to.to_owned(),
9609        reason: reason.to_owned(),
9610    });
9611}
9612
9613/// Did this reply report running a command whose own CLI never confirmed an
9614/// exit status?
9615///
9616/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9617/// command *finished* (see that type's own doc), so this can only be `true`
9618/// for a command whose completion event carried no readable exit code — not
9619/// for one that simply is not mentioned at all. That is the one signal this
9620/// crate can read, from the same record `state.jobs` renders, about a reply
9621/// standing next to work its own CLI cannot vouch for finishing; it is
9622/// deliberately not a check on the exit code's *value* (a fixer legitimately
9623/// runs a command that fails mid-iteration before it succeeds) and not a
9624/// guess at a command still running in the background (which emits no event
9625/// at all, and so leaves no evidence here to find).
9626fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9627    commands.iter().any(|c| c.exit_code.is_none())
9628}
9629
9630/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9631/// trusted as a verified no-op — the adoption guard's own text-level half.
9632///
9633/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9634/// timed out): a marker only earns the benefit of the doubt from a turn the
9635/// CLI itself vouches for finishing properly, the same house style
9636/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9637/// *fix* report to for `commands`. A candidate that timed out, exited
9638/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9639/// is, whatever prose it wrote — this returns `None` before it ever looks at
9640/// `text`. The remaining guards (the tree really is empty, the evidence is
9641/// non-empty) are the caller's: this only reads what the reply *claimed*.
9642fn verified_noop_claim(
9643    usable: bool,
9644    commands: &[agent::CommandEvidence],
9645    text: &str,
9646) -> Option<String> {
9647    (usable && !has_unconfirmed_command(commands))
9648        .then(|| verdict::verified_noop(text))
9649        .flatten()
9650}
9651
9652fn short(commit: &str) -> String {
9653    commit.chars().take(7).collect()
9654}
9655
9656fn make_executable(path: &Path) -> Result<()> {
9657    #[cfg(unix)]
9658    {
9659        use std::os::unix::fs::PermissionsExt as _;
9660        let mut perms = std::fs::metadata(path)?.permissions();
9661        perms.set_mode(0o755);
9662        std::fs::set_permissions(path, perms)?;
9663    }
9664    #[cfg(not(unix))]
9665    {
9666        let _ = path;
9667    }
9668    Ok(())
9669}
9670
9671/// What every seat in one batch shares: where the answers are attributed, the
9672/// prompt overlay they inherit, and the build cache they are told to use.
9673///
9674/// A struct rather than four more parameters: `wave` also needs the run's
9675/// state (to record who is answering right now) and the attempt number, and
9676/// eight positional arguments is both unreadable and a clippy error.
9677struct WaveCtx<'a> {
9678    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9679    /// paid for it.
9680    run: &'a str,
9681    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9682    node: &'a str,
9683    prompts: &'a Prompts,
9684    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9685    cache: Option<&'a Path>,
9686    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9687    /// `JobRecord::round`. `None` for every other node.
9688    round: Option<usize>,
9689    /// Carry each seat's failed-agent history across waves (the review loop
9690    /// only): start-of-round priority and handover choice read
9691    /// [`RunState::seat_history`], and every answer or failure writes it.
9692    carry_seats: bool,
9693    /// The run's park / interrupt requests, so a wait for a shared resource
9694    /// can be abandoned the moment the operator asks to park.
9695    pauses: Option<(&'a Pause, &'a Pause)>,
9696}
9697
9698/// Shared scheduling policy for operations and spawned seat invocations.
9699fn repository_wait(
9700    state: &RunState,
9701    pauses: Option<(&Pause, &Pause)>,
9702) -> crate::repo_lock::WaitPolicy {
9703    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9704    let inherited = crate::repo_lock::WaitPolicy::current();
9705    crate::repo_lock::WaitPolicy::new(
9706        Duration::from_secs(state.config.graph.timeout_resource_wait),
9707        move || {
9708            inherited.cancelled()
9709                || pauses
9710                    .as_ref()
9711                    .is_some_and(|(a, b)| a.parked() || b.parked())
9712        },
9713    )
9714}
9715
9716fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9717    if error.is::<crate::repo_lock::ResourceBlocked>() {
9718        state.resource_block = Some(crate::run::ResourceBlock {
9719            cause: crate::run::ResourceBlockCause::Other,
9720            node: "repository".into(),
9721            kind: if error.to_string().contains("disk pressure") {
9722                "disk pressure"
9723            } else {
9724                "repository coordination"
9725            }
9726            .into(),
9727            detail: error.to_string(),
9728            at: Timestamp::now(),
9729        });
9730        state.event("resource", error.to_string());
9731    }
9732}
9733
9734async fn coordinate_operation(
9735    state: &mut RunState,
9736    cwd: &Path,
9737    wait: &crate::repo_lock::WaitPolicy,
9738) -> Result<Option<crate::repo_lock::SeatLease>> {
9739    let result =
9740        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9741            .await;
9742    if let Err(e) = &result {
9743        record_repository_block(state, e);
9744    }
9745    result
9746}
9747
9748/// Run one job, honouring the parallelism budget.
9749async fn run_one(
9750    job: SeatJob,
9751    sem: Arc<Semaphore>,
9752    ctx: &WaveCtx<'_>,
9753    state: &mut RunState,
9754    attempt: usize,
9755) -> (SeatState, AgentOutcome) {
9756    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9757        .await
9758        .pop()
9759        .expect("one job in, one result out");
9760    (seat, out)
9761}
9762
9763/// Run every job concurrently, capped by the semaphore, preserving order.
9764///
9765/// Every seat in the batch is recorded into [`RunState::active`] before the
9766/// wave starts and cleared as each answer lands, so the run's own record says
9767/// who is still being waited on rather than only who finished.
9768async fn wave(
9769    jobs: Vec<SeatJob>,
9770    sem: Arc<Semaphore>,
9771    ctx: &WaveCtx<'_>,
9772    state: &mut RunState,
9773    attempt: usize,
9774) -> Vec<(usize, SeatState, AgentOutcome)> {
9775    let WaveCtx {
9776        run,
9777        node,
9778        prompts,
9779        cache,
9780        round,
9781        carry_seats: _,
9782        pauses,
9783    } = *ctx;
9784    for job in &jobs {
9785        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9786    }
9787    if let Err(e) = state.save() {
9788        // A failed persist of "who is answering right now" must not abort the
9789        // wave: the seats are already being asked, and the alternative is
9790        // losing the answers to save a status line nobody may even be
9791        // watching.
9792        tracing::warn!("could not persist in-progress seats: {e:#}");
9793    }
9794    // Hold the shared build cache's lease for the whole batch, not per job:
9795    // several candidates (an implement wave) or a fixer legitimately share
9796    // one cache concurrently within this run, and that stays untouched — a
9797    // single lease taken once for the whole wave and released once it is
9798    // done is what stops a *different* borrower (another run's own wave, its
9799    // e2e/gate, a human's `magi review`) from interleaving a build into the
9800    // same directory while this one is in flight. Best-effort, not
9801    // all-or-nothing: a wave that cannot get the lease within its own
9802    // longest job's budget still runs — an hour of paid implementer calls is
9803    // not thrown away over cache contention — but every write-allowed seat
9804    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9805    // below), the same fallback a read-only seat always gets, rather than
9806    // building into a directory this run was never granted. The identity
9807    // record is still invalidated below either way, so the next tracked
9808    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9809    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9810    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9811    // never borrowed from a job's agent budget (and never charged to it either,
9812    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9813    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9814    let cache_guard = if let Some(cache_dir) = cache {
9815        if jobs_had_a_writer {
9816            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9817            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9818            match acquire_cache_lease(
9819                state,
9820                cache_dir,
9821                &owner,
9822                resource_limit,
9823                node,
9824                node,
9825                &cancel,
9826            )
9827            .await
9828            {
9829                Ok(Some(g)) => Some(g),
9830                // Parking: leave without spawning anything. The run parks at
9831                // the next boundary and the seats are asked afresh on resume.
9832                Ok(None) => {
9833                    if let Some(block) = &mut state.resource_block {
9834                        block.cause = crate::run::ResourceBlockCause::Cancelled {
9835                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9836                        };
9837                    }
9838                    return block_wave(
9839                        jobs,
9840                        state,
9841                        node,
9842                        attempt,
9843                        "parked while waiting for the shared build cache",
9844                    );
9845                }
9846                // An unreadable lease is not contention: keep the old
9847                // fallback (no shared cache for this wave's seats).
9848                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9849                Err(e) => {
9850                    // Writing seats may not build into a cache this run does
9851                    // not hold, and an agent that cannot be given one is not
9852                    // asked at all: the run stops resumably instead.
9853                    let why = format!("{e:#}");
9854                    state.event(
9855                        "resource",
9856                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9857                    );
9858                    return block_wave(jobs, state, node, attempt, &why);
9859                }
9860            }
9861        } else {
9862            None
9863        }
9864    } else {
9865        None
9866    };
9867    // Every job keeps its whole budget: the wait above was not the agent's.
9868    // Restart the seats' clocks so "started" means "being spawned now".
9869    state.seats_spawning(node, attempt);
9870    if let Err(e) = state.save() {
9871        tracing::warn!("could not persist the start of a wave: {e:#}");
9872    }
9873    // Queueing behind this wave's own seats for a parallelism slot is normal
9874    // (each of them is bounded by its own agent timeout); only a wait beyond
9875    // all of them plus the resource limit means the slots never came back.
9876    let slot_limit = resource_limit
9877        + jobs
9878            .iter()
9879            .map(|j| j.timeout)
9880            .max()
9881            .unwrap_or_default()
9882            .saturating_mul(jobs.len() as u32);
9883    let mut set = tokio::task::JoinSet::new();
9884    // Seats announce themselves here the moment they hold a parallelism
9885    // permit, i.e. just before their CLI is spawned.
9886    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9887    let overlay = prompts.overlay(node);
9888    let repository_wait = repository_wait(state, pauses);
9889    for (i, mut job) in jobs.into_iter().enumerate() {
9890        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9891        if cache.is_some() {
9892            job.prompt.push('\n');
9893            job.prompt
9894                .push_str(&prompt::build_cache_note(node, job.allow_write));
9895        }
9896        let sem = Arc::clone(&sem);
9897        let run = run.to_owned();
9898        let node = node.to_owned();
9899        // Only implementers were told about the task's attachments, so only
9900        // their seats get the directory widened for reading.
9901        let attachments = if node == "implement" {
9902            state.attachments.clone()
9903        } else {
9904            Vec::new()
9905        };
9906        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9907        // `prompt::build_cache_note`'s doc for why setting it anyway is
9908        // exactly how a sandboxed reviewer's write refusal got reported as a
9909        // defect in the patch, not a property of its own seat. And a
9910        // write-allowed one is handed it only when the lease above was
9911        // actually acquired: a wave that could not get it (`cache_guard` is
9912        // `None`, see its own comment) must not send seats to build into a
9913        // directory this run does not hold - that is the exact concurrent,
9914        // unmanaged-write race this module exists to prevent, not something
9915        // "proceeding anyway" is allowed to reintroduce.
9916        let cache = cache
9917            .filter(|_| job.allow_write && cache_guard.is_some())
9918            .map(Path::to_path_buf);
9919        let spawned_tx = spawned_tx.clone();
9920        let repository_wait = repository_wait.clone();
9921        let coordination = state.config.coordination.clone();
9922        let fetched_base = state.fetched_base.clone();
9923        set.spawn(async move {
9924            // A wait for a parallelism slot is scheduling, not the agent: it
9925            // is bounded by its own limit and ends as a resource block, never
9926            // as a timeout of the agent that was not started.
9927            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9928                Ok(p) => p,
9929                Err(_) => {
9930                    return (
9931                        i,
9932                        job.seat,
9933                        AgentOutcome::ResourceBlocked(format!(
9934                            "no parallel slot became free within {}s",
9935                            slot_limit.as_secs()
9936                        )),
9937                        false,
9938                    );
9939                }
9940            };
9941            spawned_tx.send(job.seat.key.clone()).ok();
9942            let mut seat = job.seat;
9943            let out = repository_wait
9944                .scope(agent::invoke(
9945                    &job.spec,
9946                    &mut seat,
9947                    &Invocation {
9948                        coordination: coordination.as_ref(),
9949                        fetched_base: fetched_base.as_ref(),
9950                        cwd: &job.cwd,
9951                        prompt: &job.prompt,
9952                        timeout: job.timeout,
9953                        allow_write: job.allow_write,
9954                        sessions: job.sessions,
9955                        artifacts: &job.artifacts,
9956                        stem: &job.stem,
9957                        run: &run,
9958                        node: &node,
9959                        cache_dir: cache.as_deref(),
9960                        attachments: &attachments,
9961                        writable: &[],
9962                    },
9963                ))
9964                .await;
9965            let interrupted = out
9966                .as_ref()
9967                .err()
9968                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9969            let out = match out {
9970                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9971                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9972                // Billed work the CLI failed to hand over is not an ordinary
9973                // failure, but its text is the CLI's raw error JSON, not an
9974                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9975                // read it as one by forgetting to check. `usable()` is always
9976                // false here (dropped implies an empty response), so this has
9977                // to be checked before the catch-all `Failed` below or the
9978                // one shape this exists for is lost with the rest.
9979                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9980                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9981                Ok(o) => AgentOutcome::Failed(format!(
9982                    "process exited with no usable output ({})",
9983                    crate::notices::exit_status(o.exit_code)
9984                )),
9985                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9986                    AgentOutcome::ResourceBlocked(e.to_string())
9987                }
9988                Err(e) => AgentOutcome::Failed(e.to_string()),
9989            };
9990            (i, seat, out, interrupted)
9991        });
9992    }
9993    drop(spawned_tx);
9994    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9995    loop {
9996        let joined = tokio::select! {
9997            Some(key) = spawned_rx.recv() => {
9998                // The clock of this seat starts now, not when it queued.
9999                state.seat_spawned(&key);
10000                if let Err(e) = state.save() {
10001                    tracing::warn!("could not persist a seat's start: {e:#}");
10002                }
10003                continue;
10004            }
10005            joined = set.join_next() => joined,
10006        };
10007        let Some(joined) = joined else {
10008            break;
10009        };
10010        let (i, seat, out, interrupted) = match joined {
10011            Ok(v) => v,
10012            // No seat to clear: a panicked task never reported which one it
10013            // was. The defensive sweep below this loop is what stops that
10014            // seat's `active` entry from surviving forever.
10015            Err(e) => {
10016                tracing::error!("agent task panicked: {e}");
10017                continue;
10018            }
10019        };
10020        state.seat_finished(&seat.key);
10021        if interrupted
10022            && node == "implement"
10023            && let Some(label) = state
10024                .candidates
10025                .iter()
10026                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10027                .map(|candidate| candidate.label)
10028        {
10029            state.candidate_preservation.insert(
10030                label,
10031                "interrupted implementation awaiting persistence".into(),
10032            );
10033            state.interrupted_implementations.insert(label);
10034        }
10035        if let AgentOutcome::ResourceBlocked(why) = &out
10036            && state.resource_block.is_none()
10037        {
10038            state.resource_block = Some(crate::run::ResourceBlock {
10039                cause: crate::run::ResourceBlockCause::Other,
10040                node: node.to_owned(),
10041                kind: if why.contains("disk pressure") {
10042                    "disk pressure"
10043                } else if why.contains("repository coordination") {
10044                    "repository coordination"
10045                } else {
10046                    "parallel slot"
10047                }
10048                .to_owned(),
10049                detail: why.clone(),
10050                at: Timestamp::now(),
10051            });
10052            let activity = if interrupted {
10053                "agent execution interrupted"
10054            } else {
10055                "no agent was started"
10056            };
10057            state.event("resource", format!("{node}: {activity} - {why}"));
10058        }
10059        record_jobs(state, node, round, &seat.key, &out);
10060        if let Err(e) = state.save() {
10061            tracing::warn!("could not persist a seat's completion: {e:#}");
10062        }
10063        if collected.len() <= i {
10064            collected.resize_with(i + 1, || None);
10065        }
10066        collected[i] = Some((i, seat, out));
10067    }
10068    // Belt-and-braces for the panic branch above: every seat this exact batch
10069    // started shares this `(node, attempt)` pair, and every seat that finished
10070    // normally already cleared itself, so anything left tagged with it here
10071    // can only be a panicked task's leftover. Cleared unconditionally rather
10072    // than left to read as still answering forever.
10073    if state
10074        .active
10075        .values()
10076        .any(|a| a.node == node && a.attempt == attempt)
10077    {
10078        state
10079            .active
10080            .retain(|_, a| !(a.node == node && a.attempt == attempt));
10081        if let Err(e) = state.save() {
10082            tracing::warn!("could not persist the end of a wave: {e:#}");
10083        }
10084    }
10085    // Whether or not the lease above was actually held, several worktrees
10086    // may just have built into the cache with nothing here able to name one
10087    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
10088    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
10089    // clean it might not have strictly needed; trusting a stale match would
10090    // cost it a wrong answer.
10091    if let Some(cache_dir) = cache
10092        && jobs_had_a_writer
10093    {
10094        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10095    }
10096    if let Some(guard) = cache_guard {
10097        guard.release();
10098    }
10099    collected.into_iter().flatten().collect()
10100}
10101
10102/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
10103/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
10104/// this is data collection, not the fix-specific completion contract in
10105/// [`Runner::continue_fix_report`], and applies regardless of which node
10106/// asked.
10107///
10108/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
10109/// evidence from; `Failed` does not, and correctly contributes nothing — a
10110/// timeout or crash is not itself evidence about a command the seat may have
10111/// started.
10112fn record_jobs(
10113    state: &mut RunState,
10114    node: &str,
10115    round: Option<usize>,
10116    seat: &str,
10117    out: &AgentOutcome,
10118) {
10119    let commands: &[agent::CommandEvidence] = match out {
10120        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10121        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10122    };
10123    let checked_at = Timestamp::now();
10124    for c in commands {
10125        state.jobs.push(JobRecord {
10126            node: node.to_owned(),
10127            round,
10128            seat: seat.to_owned(),
10129            id: c.id.clone(),
10130            description: c.description.clone(),
10131            checked_at,
10132            status: match c.exit_code {
10133                Some(0) => JobStatus::Completed,
10134                Some(_) => JobStatus::Failed,
10135                None => JobStatus::Unknown,
10136            },
10137            exit_code: c.exit_code,
10138            result_summary: c.result_summary.clone(),
10139            source: c.source.clone(),
10140        });
10141    }
10142}
10143
10144/// Is a review round clean, given how many reviewer seats answered against
10145/// how many the round expected?
10146///
10147/// A seat that never answered (timeout, crash, unparsable output) is not a
10148/// seat that read the patch and found nothing — treating it as such is
10149/// exactly the bug this function exists to close. Under the default `block`
10150/// policy a missing seat can never be clean; `warn` still requires the seats
10151/// that *did* answer to have found nothing blocking and verification to be
10152/// green.
10153///
10154/// `quota_missing` narrows that `block` default for exactly one cause of
10155/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
10156/// a session limit lifts by the very next round buys nothing — the seat is
10157/// asked again with the same quota — so once every missing seat is accounted
10158/// for by a quota loss (and at least one seat *did* answer, so a decision has
10159/// something to rest on) the round is decided on the panel that could answer,
10160/// same as `warn` would. A panel that lost every seat to quota is not
10161/// decided here: `answered == 0` falls through to the existing `block`
10162/// fallback so a fully collapsed panel still waits rather than landing on no
10163/// review at all.
10164fn round_is_clean(
10165    blocking: usize,
10166    e2e_ok: bool,
10167    answered: usize,
10168    expected: usize,
10169    quota_missing: usize,
10170    policy: IncompleteReviewPolicy,
10171) -> bool {
10172    if blocking != 0 || !e2e_ok {
10173        return false;
10174    }
10175    if answered == expected || policy == IncompleteReviewPolicy::Warn {
10176        return true;
10177    }
10178    answered > 0 && expected - answered <= quota_missing
10179}
10180
10181/// The review loop's own conclusion, derived entirely from its persisted
10182/// round records and the round budget that produced them — never from
10183/// `status`, so a reentry (or `gate`/`merge` reading it independently)
10184/// recomputes the identical answer regardless of what an earlier node in the
10185/// same walk, or a previous walk, did to `status`.
10186///
10187/// `None` while more rounds remain to try, including when review never ran
10188/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
10189/// gone clean, or the budget is spent, or the tree has stopped moving (see
10190/// [`STAGNANT_LIMIT`]), the answer is one of two things:
10191///
10192/// - An incomplete panel that raised nothing is missing input, not a
10193///   verified tree — never a hand-off candidate, whatever verification said
10194///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
10195/// - Otherwise, green e2e on the last round hands off (see
10196///   [`Runner::stop_reviewing`]); red e2e blocks.
10197///
10198/// A last round whose own verification is still `ResourceBlocked` — magi
10199/// itself never got a command to run, not evidence the patch is broken —
10200/// is neither: this returns `None` for it too, the same as "more rounds
10201/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
10202/// handling of that shape) instead of this cheap recomputation guessing a
10203/// verdict a real attempt never produced.
10204pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10205    if max_rounds == 0
10206        || reviews.last().is_some_and(|r| {
10207            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10208        })
10209    {
10210        return Some(RunStatus::Gating);
10211    }
10212    let last = reviews.last()?;
10213    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10214    if reviews.len() < max_rounds && !stagnant {
10215        return None;
10216    }
10217    if last.incomplete() && last.blocking == 0 {
10218        return Some(RunStatus::Blocked);
10219    }
10220    if last.e2e_status() == E2eStatus::ResourceBlocked
10221        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10222    {
10223        return None;
10224    }
10225    Some(if last.e2e.iter().all(CommandOutcome::ok) {
10226        RunStatus::Gating
10227    } else {
10228        RunStatus::Blocked
10229    })
10230}
10231
10232/// How long a re-ask may take, given the budget the first attempt had.
10233///
10234/// A `nudged` retry is a request to restate an answer the seat has already
10235/// worked out: it carries no new work, so it does not deserve the original
10236/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
10237/// 133 seconds while a third sat for over ten minutes on a resumed session
10238/// holding 410 KB of prior output - and because the retry had inherited the
10239/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
10240/// judging round whose other seats were long finished.
10241///
10242/// A quarter of the budget, with a floor so that a deliberately short timeout
10243/// does not collapse to nothing. A retry that re-sends the whole prompt
10244/// (because the seat kept no context) is the original job again, and keeps the
10245/// original budget.
10246pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10247    if nudged {
10248        (full / 4).max(Duration::from_secs(120)).min(full)
10249    } else {
10250        full
10251    }
10252}
10253
10254/// Run a wave and parse each reply, re-asking the seats whose reply was
10255/// unusable.
10256///
10257/// The re-ask is a nudge rather than the whole prompt again when the seat still
10258/// holds its conversation, which is the difference between a cheap retry and
10259/// paying for the entire candidate set twice.
10260///
10261/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
10262/// successor in `roster` is handed to it instead of being re-asked: the
10263/// handover is the retry. A seat with no successor left (a single-agent
10264/// roster, the roster's tail) is nudged as before, up to `retries` times. So
10265/// the asks to one seat in one node number at most
10266/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
10267/// is asked once (a dropped stream is nudged first), and only the last agent
10268/// of the chain gets the `retries` same-agent nudges. Each roster agent is
10269/// tried at most once per seat, walking forward from the seat's own position and never wrapping
10270/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
10271/// other failure stops the chain when the previous agent failed the same way
10272/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
10273/// [`has_context`] is false and the job's own full prompt and full budget are
10274/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
10275/// intermediate ones are not losses) and is returned as a failure like any
10276/// other absent seat — the caller decides whether the panel still has a
10277/// quorum. An empty `roster` disables handover: failures are nudged as they
10278/// always were, and a quota is simply lost. A reply that fails to parse or
10279/// validate is the prompt's doing and is only ever nudged, never handed over.
10280///
10281/// The returned [`SeatState`] names the agent that answered (or tried last).
10282#[allow(clippy::too_many_arguments)]
10283async fn ask_json_wave<T>(
10284    jobs: Vec<SeatJob>,
10285    sem: Arc<Semaphore>,
10286    retries: usize,
10287    roster: &[AgentSpec],
10288    ctx: &WaveCtx<'_>,
10289    losses: &mut Vec<QuotaLoss>,
10290    state: &mut RunState,
10291    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10292) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10293where
10294    T: serde::de::DeserializeOwned + Send + 'static,
10295{
10296    ask_wave_with(
10297        jobs,
10298        sem,
10299        retries,
10300        roster,
10301        ctx,
10302        losses,
10303        state,
10304        &|text: &str| {
10305            let v = verdict::extract_json::<T>(text)?;
10306            validate(&v)?;
10307            Ok(v)
10308        },
10309    )
10310    .await
10311}
10312
10313/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
10314/// a node whose answer is prose (deliberation) shares the same handover,
10315/// failure classification, quota bookkeeping and bounds instead of a copy.
10316///
10317/// A seat handed to another roster agent is sent the job's `handover` prompt
10318/// (when it has one) rather than `prompt`: the new agent has no session, so a
10319/// resume-style prompt would be incomplete. That holds for the handover ask
10320/// and for every nudge to that agent whose `has_context` is false.
10321#[allow(clippy::too_many_arguments)]
10322async fn ask_wave_with<T>(
10323    jobs: Vec<SeatJob>,
10324    sem: Arc<Semaphore>,
10325    retries: usize,
10326    roster: &[AgentSpec],
10327    ctx: &WaveCtx<'_>,
10328    losses: &mut Vec<QuotaLoss>,
10329    state: &mut RunState,
10330    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10331) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10332where
10333    T: Send + 'static,
10334{
10335    let n = jobs.len();
10336    let originals: Vec<SeatJob> = jobs;
10337    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10338    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10339    // Nudges each seat's *current* agent has taken — 0 for a first-ask
10340    // answer, N once it has gone through N nudges. Read back once this
10341    // returns, so a caller building a history record (`ReviewRecord`) can
10342    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
10343    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
10344    // field's own doc.
10345    let mut nudges: Vec<usize> = vec![0; n];
10346    // The agent now occupying each seat, the ids it has already been through,
10347    // where in the roster the walk began, the class of the last failure, and
10348    // the stem word of a handover not yet asked (full prompt, full budget).
10349    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10350    let mut tried: Vec<BTreeSet<String>> = specs
10351        .iter()
10352        .map(|s| BTreeSet::from([s.id.clone()]))
10353        .collect();
10354    let starts: Vec<usize> = specs
10355        .iter()
10356        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10357        .collect();
10358    let carry = ctx.carry_seats && !roster.is_empty();
10359    // Carried across rounds: ids that failed the seat earlier, and how the
10360    // last failure went (so a repeat of it is not handed over again).
10361    let carried: Vec<BTreeSet<String>> = originals
10362        .iter()
10363        .map(|j| {
10364            state
10365                .seat_history
10366                .get(&j.seat.key)
10367                .filter(|_| carry)
10368                .map(|h| h.failed.clone())
10369                .unwrap_or_default()
10370        })
10371        .collect();
10372    let mut prev: Vec<Option<FailClass>> = originals
10373        .iter()
10374        .map(|j| {
10375            state
10376                .seat_history
10377                .get(&j.seat.key)
10378                .filter(|_| carry)
10379                .and_then(|h| h.last_fail.clone())
10380        })
10381        .collect();
10382    let next_agent =
10383        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10384            let others: BTreeSet<String> = specs
10385                .iter()
10386                .enumerate()
10387                .filter(|(j, _)| *j != i)
10388                .map(|(_, s)| s.id.clone())
10389                .collect();
10390            pick_successor(
10391                roster,
10392                starts[i],
10393                tried,
10394                carry.then(|| &carried[i]),
10395                &others,
10396            )
10397            .cloned()
10398        };
10399    let mut fresh: Vec<Option<String>> = vec![None; n];
10400    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10401    let mut pending: Vec<usize> = (0..n).collect();
10402
10403    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
10404    // asks: an agent with a successor is asked once and handed over, and only
10405    // a seat with no successor spends `retries` nudges on the same agent. This
10406    // only guarantees the loop's own termination whatever those say.
10407    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10408    for round in 0..max_rounds {
10409        if pending.is_empty() {
10410            break;
10411        }
10412        let mut batch = Vec::with_capacity(pending.len());
10413        let mut renudged: Vec<&str> = Vec::new();
10414        for &i in &pending {
10415            let src = &originals[i];
10416            // A seat now held by another agent than the job named has no
10417            // session of its own: it gets the full-context prompt whenever
10418            // it is asked in full (the handover ask, a nudge it cannot
10419            // resume).
10420            let full: &str = match &src.handover {
10421                Some(h) if specs[i].id != src.spec.id => h,
10422                _ => &src.prompt,
10423            };
10424            // The prompt and the budget are one decision: a nudge restates
10425            // finished work, a re-sent prompt redoes it.
10426            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10427                (
10428                    full.to_owned(),
10429                    src.timeout,
10430                    format!("{}-{word}-{}", src.stem, specs[i].id),
10431                )
10432            } else if nudges[i] == 0 {
10433                (full.to_owned(), src.timeout, src.stem.clone())
10434            } else {
10435                renudged.push(src.seat.key.as_str());
10436                let why = done[i]
10437                    .as_ref()
10438                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
10439                    .unwrap_or_else(|| "no parsable answer".to_owned());
10440                let nudge = prompt::nudge(&why);
10441                let nudged = has_context(&specs[i], &seats[i], src.sessions);
10442                let prompt = if nudged {
10443                    nudge
10444                } else {
10445                    format!("{full}\n\n---\n\n{nudge}")
10446                };
10447                (
10448                    prompt,
10449                    retry_budget(src.timeout, nudged),
10450                    format!("{}-retry{}", src.stem, nudges[i]),
10451                )
10452            };
10453            batch.push(SeatJob {
10454                spec: specs[i].clone(),
10455                seat: seats[i].clone(),
10456                cwd: src.cwd.clone(),
10457                prompt,
10458                timeout,
10459                allow_write: src.allow_write,
10460                sessions: src.sessions,
10461                artifacts: src.artifacts.clone(),
10462                stem,
10463                handover: None,
10464            });
10465        }
10466
10467        if !renudged.is_empty() {
10468            state.event(
10469                ctx.node,
10470                format!("retry {round}: re-asking {}", renudged.join(", ")),
10471            );
10472        }
10473        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10474        let mut still = Vec::new();
10475        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10476            seats[i] = seat;
10477            // No agent was asked: nothing to hand over, nudge, remember or
10478            // charge. The caller sees `resource_block` and stops the node.
10479            if let AgentOutcome::ResourceBlocked(why) = &out {
10480                done[i] = Some(Err(anyhow::anyhow!("{why}")));
10481                continue;
10482            }
10483            let class = FailClass::of(&out);
10484            // A dropped stream is nudged first (the conversation is still
10485            // there to pick up); only a seat whose nudges are spent hands over.
10486            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10487                && nudges[i] < retries
10488                && !roster.is_empty();
10489            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10490                let next = should_hand_over(prev[i].as_ref(), &cur)
10491                    .then(|| next_agent(i, &tried[i], &specs))
10492                    .flatten();
10493                if carry {
10494                    let h = state
10495                        .seat_history
10496                        .entry(originals[i].seat.key.clone())
10497                        .or_default();
10498                    h.failed.insert(specs[i].id.clone());
10499                    h.last_fail = Some(cur.clone());
10500                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10501                        h.last_ok = None;
10502                    }
10503                    // Saved before the next agent is asked, so a restart in
10504                    // between does not forget who failed.
10505                    if let Err(e) = state.save() {
10506                        tracing::warn!("could not persist a seat's failure history: {e:#}");
10507                    }
10508                }
10509                if let Some(next) = next {
10510                    record_handover(
10511                        state,
10512                        ctx.node,
10513                        &originals[i].seat.key,
10514                        &specs[i].id,
10515                        &next.id,
10516                        &cur,
10517                        &fail_reason(&out),
10518                    );
10519                    tried[i].insert(next.id.clone());
10520                    prev[i] = Some(cur.clone());
10521                    seats[i] =
10522                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10523                    specs[i] = next;
10524                    fresh[i] = Some(cur.stem_word().to_owned());
10525                    nudges[i] = 0;
10526                    done[i] = Some(Err(anyhow::anyhow!(
10527                        "handed over after: {}",
10528                        fail_reason(&out)
10529                    )));
10530                    still.push(i);
10531                    continue;
10532                }
10533            }
10534            if carry
10535                && !nudge_first
10536                && let Some(cur) = class.clone()
10537            {
10538                // The chain ended here (no successor, or a repeated failure).
10539                let h = state
10540                    .seat_history
10541                    .entry(originals[i].seat.key.clone())
10542                    .or_default();
10543                h.failed.insert(specs[i].id.clone());
10544                h.last_fail = Some(cur);
10545            }
10546            let parsed = match out {
10547                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10548                AgentOutcome::Quota(o) => {
10549                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10550                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10551                }
10552                // Not a parseable answer: the nudge loop re-asks it, which is
10553                // exactly what a dropped stream needs. Just don't hand its raw
10554                // error JSON to `extract_json`.
10555                AgentOutcome::Dropped(o) => {
10556                    let why = o
10557                        .dropped
10558                        .as_ref()
10559                        .map(|d| d.why.as_str())
10560                        .unwrap_or("the CLI ended the stream without delivering its answer");
10561                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10562                }
10563                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10564                    Err(anyhow::anyhow!(e))
10565                }
10566            };
10567            let quota = class == Some(FailClass::Quota);
10568            let failed = parsed.is_err();
10569            done[i] = Some(parsed);
10570            if carry && !failed {
10571                let h = state
10572                    .seat_history
10573                    .entry(originals[i].seat.key.clone())
10574                    .or_default();
10575                h.failed.remove(&specs[i].id);
10576                h.last_ok = Some(specs[i].id.clone());
10577                h.last_fail = None;
10578            }
10579            // Do not re-ask a rate-limited seat (quota) — a retry is known to
10580            // fail the same way; and never re-ask a seat that already parsed.
10581            // A failed agent that still has a successor is not re-asked
10582            // either: the handover was its remedy and has just been refused
10583            // (the chain stops on a repeated failure class). A seat with no
10584            // successor left (a single-agent roster, the roster's tail, or an
10585            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10586            let agent_failure = class.is_some()
10587                && !nudge_first
10588                && !roster.is_empty()
10589                && next_agent(i, &tried[i], &specs).is_some();
10590            if failed && !quota && !agent_failure && nudges[i] < retries {
10591                nudges[i] += 1;
10592                still.push(i);
10593            }
10594        }
10595        pending = still;
10596    }
10597
10598    // One loss per seat whose chain ended on a quota: the intermediate ones
10599    // were absorbed by a handover and are not losses.
10600    for (i, q) in last_quota.into_iter().enumerate() {
10601        if let Some(reset) = q {
10602            losses.push(QuotaLoss {
10603                seat: originals[i].seat.key.clone(),
10604                node: ctx.node.to_owned(),
10605                at: Timestamp::now(),
10606                reset,
10607            });
10608        }
10609    }
10610
10611    seats
10612        .into_iter()
10613        .zip(done)
10614        .zip(nudges)
10615        .map(|((seat, res), attempts)| {
10616            (
10617                seat,
10618                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10619                attempts,
10620            )
10621        })
10622        .collect()
10623}
10624
10625/// Did the wave end without asking any agent (see [`block_wave`])?
10626fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10627    results
10628        .iter()
10629        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10630}
10631
10632/// Settle a wave that never spawned anything: every job answers
10633/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10634/// else about the seat (session, history, quota) is touched.
10635fn block_wave(
10636    jobs: Vec<SeatJob>,
10637    state: &mut RunState,
10638    node: &str,
10639    attempt: usize,
10640    why: &str,
10641) -> Vec<(usize, SeatState, AgentOutcome)> {
10642    state.resource_wait = None;
10643    state
10644        .active
10645        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10646    if let Err(e) = state.save() {
10647        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10648    }
10649    jobs.into_iter()
10650        .enumerate()
10651        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10652        .collect()
10653}
10654
10655/// Acquire the shared build cache's lease, waiting out contention within
10656/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10657/// unbounded wait is never acceptable).
10658///
10659/// A first, non-blocking check happens before ever waiting; if it finds the
10660/// lease busy, that fact is logged as a `verify` event *and* flushed with
10661/// [`RunState::save`] immediately — not only once the wait finally succeeds
10662/// or gives up — so a `magi show` run by a different process while this one
10663/// is still waiting reads a `run.json` that says so, rather than whatever it
10664/// looked like before the wait started. The same applies to the terminal
10665/// failure: logged and saved before this returns `Err`, so a caller that
10666/// could not get the lease at all still leaves a legible record of why.
10667async fn acquire_cache_lease(
10668    state: &mut RunState,
10669    cache_dir: &Path,
10670    owner: &crate::cache::Owner,
10671    budget: Duration,
10672    node: &str,
10673    context: &str,
10674    cancel: &(dyn Fn() -> bool + Sync),
10675) -> Result<Option<crate::cache::Guard>> {
10676    let home = crate::run::home();
10677    let started = Instant::now();
10678    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10679        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10680        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10681        Err(e) => {
10682            state.event(
10683                "verify",
10684                format!("{context}: could not check the shared build cache: {e:#}"),
10685            );
10686            if let Err(e2) = state.save() {
10687                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10688            }
10689            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10690        }
10691    };
10692    state.event(
10693        "verify",
10694        format!(
10695            "{context}: waiting for the shared build cache at {} ({})",
10696            cache_dir.display(),
10697            busy.describe()
10698        ),
10699    );
10700    // Seats of this node are waiting, not running: say so, so the countdown a
10701    // reader sees only ever measures an agent that has been spawned.
10702    state.resource_wait = Some(crate::run::ResourceWait {
10703        node: context.to_owned(),
10704        kind: "shared build cache".to_owned(),
10705        holder: busy.describe(),
10706        since: Timestamp::now(),
10707        limit_secs: budget.as_secs(),
10708    });
10709    if let Err(e) = state.save() {
10710        tracing::warn!("could not persist a cache wait: {e:#}");
10711    }
10712    let remaining = budget.saturating_sub(started.elapsed());
10713    let waited = crate::cache::wait_for_unless(
10714        &home,
10715        cache_dir,
10716        owner,
10717        remaining,
10718        Duration::from_secs(5),
10719        cancel,
10720    )
10721    .await;
10722    state.resource_wait = None;
10723    let cause = match &waited {
10724        Ok(Some(_)) => None,
10725        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10726        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10727            Some(crate::run::ResourceBlockCause::Timeout {
10728                waited_secs: started.elapsed().as_secs(),
10729                limit_secs: budget.as_secs(),
10730            })
10731        }
10732        Err(_) => Some(crate::run::ResourceBlockCause::Other),
10733    };
10734    if let Some(cause) = cause {
10735        let detail = match &waited {
10736            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10737            Err(error) => format!("{error:#}"),
10738        };
10739        state.resource_block = Some(crate::run::ResourceBlock {
10740            cause,
10741            node: node.to_owned(),
10742            kind: "shared build cache".into(),
10743            detail,
10744            at: Timestamp::now(),
10745        });
10746        state.event(
10747            "resource",
10748            state.resource_block.as_ref().unwrap().description(),
10749        );
10750    }
10751    if let Err(error) = state.save() {
10752        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10753    }
10754    waited
10755}
10756
10757/// The lease state could not be read at all (unreadable lease file, a
10758/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10759/// known to be contended, so a wave degrades to running without a shared cache
10760/// instead of stopping on a hold that [`crate::triage`] could never release.
10761#[derive(Debug)]
10762struct CacheCheckError(String);
10763
10764impl std::fmt::Display for CacheCheckError {
10765    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10766        f.write_str(&self.0)
10767    }
10768}
10769
10770impl std::error::Error for CacheCheckError {}
10771
10772/// Keep a specific acquisition outcome when the caller adds command evidence.
10773fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10774    state
10775        .resource_block
10776        .get_or_insert_with(|| crate::run::ResourceBlock {
10777            cause: crate::run::ResourceBlockCause::Other,
10778            node: node.to_owned(),
10779            kind: "shared build cache".into(),
10780            detail: format!("{error:#}"),
10781            at: Timestamp::now(),
10782        });
10783    if let Err(error) = state.save() {
10784        tracing::warn!("could not persist cache resource block: {error:#}");
10785    }
10786}
10787
10788/// Run `body` — a verify command batch — while holding the shared build
10789/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10790/// never interleave with another borrower's build against the same
10791/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10792/// timeout, or a human's own `magi review`. See the `cache` module doc for
10793/// why this matters more than Cargo's own per-target locking covers — two
10794/// *different* worktrees building the same package name/version into one
10795/// cache directory is a staleness bug, not a lock contention one.
10796///
10797/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10798/// agent waves do. Repository scheduling also has its own bounded,
10799/// cancellable resource-wait policy. Contention must not consume the
10800/// verification budget: `body` receives the full `budget` after both leases
10801/// are acquired and the cache is fresh. Without a shared cache, only
10802/// repository coordination is acquired before handing over the full budget.
10803///
10804/// A lease that cannot be acquired within the resource wait is reported as a single
10805/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10806/// verification — the same shape a spawn failure already takes in
10807/// [`run_commands`], so a caller need not special-case it.
10808#[allow(clippy::too_many_arguments)]
10809async fn with_cache_lease<'s, F, Fut>(
10810    state: &'s mut RunState,
10811    cache_dir: Option<&Path>,
10812    node: &str,
10813    seat: &str,
10814    worktree: &Path,
10815    head: &str,
10816    budget: Duration,
10817    context: &str,
10818    wait: &crate::repo_lock::WaitPolicy,
10819    body: F,
10820) -> Result<(Vec<CommandOutcome>, bool)>
10821where
10822    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10823    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10824{
10825    // Match editing waves: cache first, repository second. In particular,
10826    // pre_gate must never hold the repository while waiting for a wave's cache.
10827    let started = Instant::now();
10828    let home = crate::run::home();
10829    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10830    let guard = if let Some(cache_dir) = cache_dir {
10831        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10832        match acquire_cache_lease(
10833            state,
10834            cache_dir,
10835            &owner,
10836            resource_limit,
10837            node,
10838            context,
10839            &|| wait.cancelled(),
10840        )
10841        .await
10842        .and_then(|g| {
10843            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10844        }) {
10845            Ok(g) => Some(g),
10846            Err(e) => {
10847                record_cache_error(state, node, &e);
10848                return Ok((
10849                    vec![CommandOutcome {
10850                        command: "(waiting for the shared build cache)".into(),
10851                        code: None,
10852                        output_tail: e.to_string(),
10853                        duration_ms: started.elapsed().as_millis() as u64,
10854                        resource_blocked: true,
10855                    }],
10856                    false,
10857                ));
10858            }
10859        }
10860    } else {
10861        None
10862    };
10863    let _coordination = coordinate_operation(state, worktree, wait).await?;
10864    if let Some(cache_dir) = cache_dir {
10865        let identity = crate::cache::Identity::new(worktree, head);
10866        if let Err(e) = crate::cache::ensure_fresh(
10867            &home,
10868            cache_dir,
10869            &identity,
10870            &state.config.verify.cargo_manifests(),
10871        ) {
10872            record_cache_error(state, node, &e);
10873            state.event(
10874                "verify",
10875                format!(
10876                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10877                    worktree.display(),
10878                    short(head)
10879                ),
10880            );
10881            if let Some(guard) = guard {
10882                guard.release();
10883            }
10884            return Ok((
10885                vec![CommandOutcome {
10886                    command: "(confirming the shared build cache is fresh)".into(),
10887                    code: None,
10888                    output_tail: e.to_string(),
10889                    duration_ms: started.elapsed().as_millis() as u64,
10890                    resource_blocked: true,
10891                }],
10892                false,
10893            ));
10894        }
10895    }
10896    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10897    if !timed_out_pids.is_empty() {
10898        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10899    }
10900    // Drop repository coordination before releasing the cache, reversing the
10901    // acquisition order. A cancelled/error future also drops both guards.
10902    drop(_coordination);
10903    if let Some(guard) = guard {
10904        guard.release();
10905    }
10906    Ok((outcomes, retried))
10907}
10908
10909/// Poll `pids` — commands [`run_commands`] reports as still running when its
10910/// own timeout elapsed — until every one is confirmed gone, or
10911/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10912///
10913/// Real confirmation where confirmation is possible, not a substitute for
10914/// full process-tree observation: a grandchild the timed-out process spawned
10915/// and that survives independently of it is invisible to a pid check the
10916/// same way it always was, and continuing to observe and collect *that*
10917/// stays a different piece of work with its own owner. This only narrows a
10918/// fixed blind wait into an actual check of the pids this process does know
10919/// about.
10920async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10921    wait_for_pids_with(
10922        pids,
10923        crate::proc::pid_alive,
10924        LEASE_RELEASE_POLL,
10925        LEASE_RELEASE_MAX_WAIT,
10926    )
10927    .await;
10928}
10929
10930/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10931/// interval and ceiling supplied by the caller, so the polling *logic* -
10932/// returns as soon as every pid reports dead, gives up at the ceiling
10933/// otherwise - is testable on millisecond durations without asking the real
10934/// OS about a pid at all.
10935async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10936    pids: &[u32],
10937    alive: F,
10938    poll: Duration,
10939    max_wait: Duration,
10940) {
10941    let deadline = Instant::now() + max_wait;
10942    loop {
10943        if pids.iter().all(|&pid| !alive(pid)) {
10944            return;
10945        }
10946        if Instant::now() >= deadline {
10947            return;
10948        }
10949        tokio::time::sleep(poll).await;
10950    }
10951}
10952
10953/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10954/// admission that it could not even get a verify command to run, as opposed
10955/// to evidence the command actually produced? A caller that would otherwise
10956/// read a resource-blocked outcome as a red command must check this first:
10957/// see [`Runner::gate`], which retries rather than records `Blocked` when
10958/// this is true.
10959fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10960    outcomes.iter().any(|o| o.resource_blocked)
10961}
10962
10963/// What [`Runner::gate_fix_round`] decided.
10964pub(crate) enum GateFix {
10965    /// The tree changed and `verify.e2e` is still green: run the gate again.
10966    Retry,
10967    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10968    /// last failure stands and the run ends blocked.
10969    Stop,
10970    /// `verify.e2e` could not run after the fix (magi's own contention):
10971    /// decide nothing now, a later reentry retries.
10972    Defer,
10973}
10974
10975/// Is every red command in `outcomes` an ordinary failure the code could
10976/// explain: it ran, exited non-zero, and said something?
10977///
10978/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10979/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10980/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10981/// what the command is stays the gate's business.
10982fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10983    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10984    red.peek().is_some()
10985        && red.all(|o| {
10986            !o.resource_blocked
10987                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10988                && !o.output_tail.trim().is_empty()
10989        })
10990}
10991
10992/// Describe one verify command's outcome for the event log, distinguishing a
10993/// build/link failure — the toolchain never produced a binary to run — from
10994/// an actual test failure, since only the latter is a verdict on the patch.
10995fn e2e_outcome_label(o: &CommandOutcome) -> String {
10996    if o.ok() {
10997        return "pass".to_owned();
10998    }
10999    let reason = if o.build_failed() {
11000        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11001    } else {
11002        format!("FAIL ({:?})", o.code)
11003    };
11004    format!(
11005        "{reason}\n{}",
11006        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11007    )
11008}
11009
11010/// Run `verify.e2e`, retrying once if the first attempt could not build or
11011/// link — a build/link failure is frequently a race against a shared
11012/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
11013/// `verify` event per command, tagged with `context` (normally `"round N"`)
11014/// so the two call sites that need this — the ordinary per-round leg in
11015/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
11016/// it will ever call a round green — read identically in the event log.
11017async fn run_e2e_with_retry(
11018    state: &mut RunState,
11019    shell: &[String],
11020    commands: &[String],
11021    worktree: &Path,
11022    timeout: Duration,
11023    context: &str,
11024) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11025    let (mut e2e, mut timed_out_pids) = run_commands(
11026        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11027    )
11028    .await;
11029    for o in &e2e {
11030        state.event(
11031            "verify",
11032            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11033        );
11034    }
11035    // A build/link failure is not a verdict on the patch — it is frequently a
11036    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
11037    // one retry before letting a red like that decide the round.
11038    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11039    if verify_retried {
11040        state.event(
11041            "verify",
11042            format!(
11043                "{context}: verify could not build/link, not a test result — retrying once \
11044                 before concluding"
11045            ),
11046        );
11047        let retried = run_commands(
11048            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11049        )
11050        .await;
11051        e2e = retried.0;
11052        // Both attempts' timeouts matter, not just the last one: the first
11053        // attempt's descendants may still be alive alongside the retry's.
11054        timed_out_pids.extend(retried.1);
11055        for o in &e2e {
11056            state.event(
11057                "verify",
11058                format!(
11059                    "{context}: retry `{}` -> {}",
11060                    o.command,
11061                    e2e_outcome_label(o)
11062                ),
11063            );
11064        }
11065    }
11066    (e2e, verify_retried, timed_out_pids)
11067}
11068
11069/// Run configured shell commands in `cwd`, in order. The second element is
11070/// the pid of every command that hit `timeout` and was still running when
11071/// this stopped waiting on it (best-effort: `None` when the platform did not
11072/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
11073/// that releases a shared resource afterward needs to know.
11074///
11075/// Records `task` into [`RunState::active`] at every command boundary
11076/// (`RunState::task_command`) and clears it once the whole list has run
11077/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
11078/// for minutes with no seat and no output of its own to show for it (see
11079/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
11080/// apart from "not yet run" without this), and this is the only place that
11081/// knows which command is running right now and how many are left. Three
11082/// saves per command — start, not per second — matching the same "only at a
11083/// boundary" rule [`wave`] already follows for seats.
11084#[allow(clippy::too_many_arguments)]
11085async fn run_commands(
11086    state: &mut RunState,
11087    node: &str,
11088    task: &str,
11089    attempt: usize,
11090    shell: &[String],
11091    commands: &[String],
11092    cwd: &Path,
11093    timeout: Duration,
11094) -> (Vec<CommandOutcome>, Vec<u32>) {
11095    if commands.is_empty() {
11096        // Nothing to mark as running and nothing to clear — an empty list
11097        // means "not configured", and touching `active` (or the disk) over
11098        // that would be a write for every round of a repo with no
11099        // `verify.e2e` / `verify.gate` commands at all.
11100        return (Vec::new(), Vec::new());
11101    }
11102    let mut out = Vec::new();
11103    let mut timed_out_pids = Vec::new();
11104    let total = commands.len();
11105    for (idx, command) in commands.iter().enumerate() {
11106        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11107        if let Err(e) = state.save() {
11108            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11109        }
11110        let started = Instant::now();
11111        let mut cmd = tokio::process::Command::new(&shell[0]);
11112        cmd.quiet();
11113        // Verify the checkout as a standalone caller. Worker ownership and
11114        // pinned executable selection belong to this driver, not to tests or
11115        // tools it runs. Preserve home/seat context and Running's containment.
11116        for key in crate::worker::INVOCATION_ENV {
11117            cmd.env_remove(key);
11118        }
11119        cmd.args(&shell[1..])
11120            .arg(command)
11121            .current_dir(cwd)
11122            .stdin(std::process::Stdio::null())
11123            .stdout(std::process::Stdio::piped())
11124            .stderr(std::process::Stdio::piped())
11125            .kill_on_drop(true);
11126        let mut outputs = Vec::new();
11127        let protected = (|| -> Result<()> {
11128            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
11129                outputs.push(guard);
11130            }
11131            if let Some(cache) = state.config.cache_dir() {
11132                outputs.push(crate::retention::OutputLease::cache(
11133                    &crate::run::home(),
11134                    &cache,
11135                    false,
11136                )?);
11137            }
11138            Ok(())
11139        })();
11140        if let Err(e) = protected {
11141            out.push(CommandOutcome {
11142                command: command.clone(),
11143                code: None,
11144                output_tail: format!("output ownership unavailable: {e:#}"),
11145                duration_ms: 0,
11146                resource_blocked: true,
11147            });
11148            continue;
11149        }
11150        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
11151        let (code, body) = match spawned {
11152            Ok(mut running) => {
11153                let pid = running.child().id();
11154                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
11155                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
11156                let mut paths = vec![cwd.to_owned()];
11157                if let Some(cache) = state.config.cache_dir() {
11158                    paths.push(cache);
11159                }
11160                let mut pressure = None;
11161                let status = tokio::select! {
11162                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
11163                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
11164                        pressure = Some(reason); None
11165                    }
11166                };
11167                let (code, failure) = match status {
11168                    Some(Ok(Ok(status))) => (
11169                        status.code(),
11170                        status
11171                            .code()
11172                            .is_none()
11173                            .then(|| format!("command terminated: {status}")),
11174                    ),
11175                    Some(Ok(Err(error))) => {
11176                        (None, Some(format!("failed waiting for command: {error}")))
11177                    }
11178                    Some(Err(_)) => {
11179                        // Leased cleanup confirms process-tree death itself.
11180                        // Unleased commands still need the caller's PID check
11181                        // after kill-on-drop, before releasing cache ownership.
11182                        if !running.coordinated()
11183                            && let Some(pid) = pid
11184                        {
11185                            timed_out_pids.push(pid);
11186                        }
11187                        (
11188                            None,
11189                            Some(format!("timed out after {}s", timeout.as_secs())),
11190                        )
11191                    }
11192                    None => (None, None),
11193                };
11194                running.finish().await;
11195                if let Some(reason) = pressure {
11196                    state.resource_block = Some(crate::run::ResourceBlock {
11197                        cause: crate::run::ResourceBlockCause::Other,
11198                        node: node.to_owned(),
11199                        kind: "disk pressure".to_owned(),
11200                        detail: reason.clone(),
11201                        at: Timestamp::now(),
11202                    });
11203                    out.push(CommandOutcome {
11204                        command: command.clone(),
11205                        code: None,
11206                        output_tail: reason,
11207                        duration_ms: started.elapsed().as_millis() as u64,
11208                        resource_blocked: true,
11209                    });
11210                    break;
11211                }
11212                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11213                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11214                if let Some(failure) = failure {
11215                    body.push('\n');
11216                    body.push_str(&failure);
11217                }
11218                (code, body)
11219            }
11220            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11221        };
11222        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
11223        if !body.is_empty() {
11224            // The full text is kept whatever the summary dropped. A fresh
11225            // name per command so a resume or retry never overwrites an
11226            // earlier run's output. A failed write changes no verdict.
11227            match crate::run::write_verification_artifact(state, task, &body) {
11228                Ok(name) if summary != body => {
11229                    summary = format!("[full output: artifacts/{name}]\n{summary}");
11230                }
11231                Ok(_) => {}
11232                Err(e) => {
11233                    let warning = format!("could not save the full {task} output: {e:#}");
11234                    tracing::warn!("{warning}");
11235                    state.event(node, warning);
11236                    if summary != body {
11237                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
11238                    }
11239                }
11240            }
11241        }
11242        out.push(CommandOutcome {
11243            command: command.clone(),
11244            code,
11245            output_tail: summary,
11246            duration_ms: started.elapsed().as_millis() as u64,
11247            resource_blocked: false,
11248        });
11249    }
11250    state.task_finished(task);
11251    if let Err(e) = state.save() {
11252        tracing::warn!("could not persist the end of {task}: {e:#}");
11253    }
11254    (out, timed_out_pids)
11255}
11256
11257/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
11258/// section (`report::run`) and in the `merge` event this node records — for
11259/// the operator to run by hand.
11260///
11261/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
11262/// branch whose ruleset forbids merge commits (GitHub's "must not contain
11263/// merge commits", or "require linear history") rejects the push a `--no-ff`
11264/// merge would produce, which is exactly the guidance this function replaces.
11265/// `message`'s first line becomes the squash commit's subject, matching the
11266/// note `report::run` prints alongside this command — see that function for
11267/// why an explicit subject is not optional there.
11268fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11269    let repo = repo.display();
11270    match style {
11271        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11272        MergeStyle::Squash => {
11273            // The subject sits inside double quotes, and a title an agent
11274            // wrote may carry the characters that break out of them.
11275            let subject = message
11276                .lines()
11277                .next()
11278                .unwrap_or(branch)
11279                .replace(['\\', '"', '$', '`'], "");
11280            format!(
11281                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11282            )
11283        }
11284        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11285    }
11286}
11287
11288/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
11289/// under the hood, rejects a `title` over 256 characters and the whole
11290/// command fails — no PR at all, for a run whose body was otherwise fine
11291/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
11292/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
11293/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
11294/// this bound. It is a margin, not a guarantee — a title packed
11295/// with multi-unit characters could still in principle land close to the
11296/// edge, but a real task title's occasional emoji or accented letter fits
11297/// comfortably inside it.
11298const PR_TITLE_MAX: usize = 240;
11299
11300/// A pull request title from the opening line of `text`, or `None` when that
11301/// line is not English (GitHub text is) or has no letters.
11302///
11303/// A line within `max` is kept as is. A longer one is cut at the end of its
11304/// first sentence when that falls inside `max`, else at a word boundary with a
11305/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
11306/// make every merely-truncated title look non-English). The language check
11307/// runs on the kept text before any mark is added, so only what GitHub will
11308/// show is judged: an English opening followed by non-ASCII far past the cut
11309/// still passes.
11310fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11311    let line = queue::first_line(text)?;
11312    let chars: Vec<char> = line.chars().collect();
11313    let (kept, mark) = if chars.len() <= max {
11314        (line.to_owned(), "")
11315    } else if let Some(end) = sentence_end(&chars, max) {
11316        (chars[..end].iter().collect::<String>(), "")
11317    } else {
11318        let room = max.saturating_sub(3);
11319        // Cut at the last space inside the room; when the char just past the
11320        // room is a space the room already ends on a word.
11321        let cut = if chars[room].is_whitespace() {
11322            room
11323        } else {
11324            chars[..room]
11325                .iter()
11326                .rposition(|c| c.is_whitespace())
11327                .unwrap_or(room)
11328        };
11329        let head: String = chars[..cut].iter().collect();
11330        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11331        (head.to_owned(), "...")
11332    };
11333    if kept.is_empty() {
11334        return None;
11335    }
11336    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11337        return None;
11338    }
11339    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11340        return None;
11341    }
11342    Some(format!("{kept}{mark}"))
11343}
11344
11345/// The char length of the first sentence of `chars` when it ends within `max`
11346/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
11347/// abbreviations so `e.g. foo` does not end a title early.
11348fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11349    const MIN: usize = 20;
11350    for i in MIN..max.min(chars.len()) {
11351        if !matches!(chars[i], '.' | '!' | '?') {
11352            continue;
11353        }
11354        let Some(&next) = chars.get(i + 1) else {
11355            continue;
11356        };
11357        if !next.is_whitespace() {
11358            continue;
11359        }
11360        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11361        if after.is_some_and(|c| c.is_ascii_lowercase()) {
11362            continue;
11363        }
11364        let word: String = chars[..i]
11365            .iter()
11366            .rev()
11367            .take_while(|c| !c.is_whitespace())
11368            .collect::<Vec<_>>()
11369            .into_iter()
11370            .rev()
11371            .collect();
11372        let word = word.to_ascii_lowercase();
11373        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11374            continue;
11375        }
11376        let end = chars[..i]
11377            .iter()
11378            .rposition(|c| !c.is_whitespace())
11379            .map_or(i, |p| p + 1);
11380        return Some(end);
11381    }
11382    None
11383}
11384
11385/// What `merge = "pr"` (and the merge commit of the other modes) says about a
11386/// change: a title and a body describing what was *implemented*, not the task
11387/// that asked for it. A task reads as a request; a reader of the merged
11388/// history wants the change.
11389struct PrMessage {
11390    title: String,
11391    body: String,
11392}
11393
11394impl PrMessage {
11395    /// Title, blank line, body. The first line is the squash/merge commit
11396    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
11397    /// has to stay one sensible line.
11398    fn commit_message(&self) -> String {
11399        format!("{}\n\n{}", self.title, self.body)
11400    }
11401}
11402
11403/// The text after a leading `TITLE:` (any case) on `line`.
11404fn title_marker(line: &str) -> Option<&str> {
11405    let line = line.trim();
11406    let head = line.get(..6)?;
11407    head.eq_ignore_ascii_case("title:")
11408        .then(|| line[6..].trim())
11409}
11410
11411/// The implementer's own one-line title: the `TITLE:` line the implement
11412/// prompt asks for at the top of its SUMMARY. Candidate commits are all
11413/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
11414/// source, and a title that says as much is refused here too.
11415fn summary_title(summary: &str) -> Option<String> {
11416    let first = summary.lines().find(|l| !l.trim().is_empty())?;
11417    let raw = title_marker(first)?;
11418    if raw.is_empty() {
11419        return None;
11420    }
11421    let title = queue::title_from(raw, PR_TITLE_MAX);
11422    let lower = title.to_ascii_lowercase();
11423    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11424        return None;
11425    }
11426    Some(title)
11427}
11428
11429/// How `open_review`'s instruction begins; see [`landing_title`].
11430const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11431
11432/// Marker `open_review` gives a candidate that nothing in the roster wrote.
11433pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11434
11435/// Does this run review work that already existed, rather than implement a
11436/// task? Runs recorded before `reviewed_commits` existed carry only the
11437/// candidate marker.
11438pub(crate) fn is_review_run(state: &RunState) -> bool {
11439    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11440}
11441
11442/// The title of a review-only run: the subject of the oldest commit under
11443/// review. Later commits are usually fixups, and `instruction` is the review
11444/// prompt, which says nothing about the change. GitHub text is English, so a
11445/// non-ASCII or blank subject yields `None` and the caller's neutral title.
11446fn review_title(state: &RunState) -> Option<String> {
11447    english_subject(
11448        state.reviewed_commits.as_ref()?.first()?,
11449        state.config.graph.github_english(),
11450    )
11451}
11452
11453/// `raw` as a pull request title, or `None` when it is blank, not English
11454/// (GitHub text is), or one of magi's own candidate commit subjects.
11455fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11456    let raw = raw.trim();
11457    if raw.is_empty() {
11458        return None;
11459    }
11460    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11461        return None;
11462    }
11463    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11464        return None;
11465    }
11466    let title = queue::title_from(raw, PR_TITLE_MAX);
11467    let lower = title.to_ascii_lowercase();
11468    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11469        return None;
11470    }
11471    Some(title)
11472}
11473
11474/// What a review-only run's branch says about itself, read at the moment the
11475/// pull request is opened.
11476#[derive(Debug, Clone, PartialEq, Eq)]
11477struct BranchFacts {
11478    /// `(subject, body)` of each commit, oldest first.
11479    commits: Vec<(String, String)>,
11480    /// Trimmed `git diff --stat`.
11481    stat: String,
11482}
11483
11484/// Longest diff stat shown: this many file lines plus the summary line.
11485const STAT_FILE_LINES: usize = 25;
11486/// Cap on the commit list, well inside GitHub's 65536-character body limit.
11487const COMMITS_MAX_CHARS: usize = 20_000;
11488
11489/// Read the commits and diff stat of `base..branch`. `None` when git cannot
11490/// say or finds nothing, so the caller falls back to what the run recorded.
11491async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11492    let commits = git::commit_log(repo, base, branch).await.ok()?;
11493    if commits.is_empty() {
11494        return None;
11495    }
11496    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11497    let lines: Vec<&str> = stat.lines().collect();
11498    let stat = if lines.len() > STAT_FILE_LINES + 1 {
11499        let omitted = lines.len() - 1 - STAT_FILE_LINES;
11500        let more = format!(" ... {omitted} more file(s)");
11501        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11502        kept.push(&more);
11503        kept.push(lines[lines.len() - 1]);
11504        kept.join("\n")
11505    } else {
11506        lines.join("\n")
11507    };
11508    Some(BranchFacts { commits, stat })
11509}
11510
11511/// Defang what would break the surrounding markdown: a closing `</details>`
11512/// and a code fence.
11513fn markdown_safe(text: &str) -> String {
11514    text.replace("</details>", "&lt;/details&gt;")
11515        .replace("\x60\x60\x60", "~~~")
11516}
11517
11518fn neutral_title(state: &RunState, winner: char) -> String {
11519    format!(
11520        "chore: land candidate {} of run {}",
11521        winner.to_ascii_uppercase(),
11522        state.id
11523    )
11524}
11525
11526/// Reuse the graph's bounded fixer without starting another graph or driver.
11527/// Copy the working state back even on errors so spent rounds remain visible.
11528pub(crate) async fn repair_landing_gate(
11529    state: &mut RunState,
11530    winner: &Candidate,
11531    outcomes: &[CommandOutcome],
11532) -> Result<GateFix> {
11533    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11534        return Ok(GateFix::Stop);
11535    }
11536    let mut runner = Runner {
11537        roles: state.config.resolve_roles()?,
11538        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11539        state: state.clone(),
11540        pause: Pause::new(),
11541        interrupt: Pause::new(),
11542    };
11543    let result = runner.gate_fix_round(winner, outcomes).await;
11544    *state = runner.state;
11545    result
11546}
11547
11548/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
11549/// A completed failure is terminal on that SHA. Resource waits remain retryable.
11550pub(crate) async fn verify_current_head(
11551    state: &mut RunState,
11552    winner: &Candidate,
11553    wait: &crate::repo_lock::WaitPolicy,
11554) -> Result<bool> {
11555    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11556    // Preserve the previous record before replacing it, including on crash recovery.
11557    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
11558        state
11559            .verification_history
11560            .push(crate::run::VerificationEvidence {
11561                superseded_by: Some(head.clone()),
11562                archived_at: Some(Timestamp::now()),
11563                fixes_spent: Some(state.gate_fixes.len()),
11564                gate_head: None,
11565                gate_ran: false,
11566                gate: Vec::new(),
11567                e2e: state.head_e2e.take(),
11568            });
11569    }
11570    if let Some(e) = &state.head_e2e
11571        && e.head == head
11572    {
11573        if e.passed {
11574            return Ok(true);
11575        }
11576        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
11577            state.status = RunStatus::Blocked;
11578            state.save()?;
11579            return Ok(false);
11580        }
11581    } else if state.reviews.last().is_some_and(|r| {
11582        r.verified_head.as_deref() == Some(head.as_str())
11583            && (r.e2e_status() == E2eStatus::Passed
11584                || (r.e2e_status() == E2eStatus::NotConfigured
11585                    && state.config.verify.e2e.is_empty()))
11586    }) || state
11587        .gate_repair_e2e
11588        .as_ref()
11589        .is_some_and(|e| e.head == head && e.passed)
11590    {
11591        return Ok(true);
11592    }
11593    if let Some(e) = &state.gate_repair_e2e
11594        && e.head == head
11595        && !e.passed
11596        && !e.outcomes.is_empty()
11597        && !verify_inconclusive(&e.outcomes)
11598    {
11599        state.status = RunStatus::Blocked;
11600        state.save()?;
11601        return Ok(false);
11602    }
11603    state.head_e2e = Some(crate::run::GateRepairE2e {
11604        head: head.clone(),
11605        passed: false,
11606        outcomes: Vec::new(),
11607    });
11608    state.save()?;
11609    let commands = state.config.verify.e2e.clone();
11610    let shell = state.config.shell();
11611    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11612    let cache_dir = state.config.cache_dir();
11613    let outcomes = if commands.is_empty() {
11614        Vec::new()
11615    } else {
11616        let (outcomes, _) = with_cache_lease(
11617            state,
11618            cache_dir.as_deref(),
11619            "e2e",
11620            "e2e",
11621            &winner.worktree,
11622            &head,
11623            timeout,
11624            "current head e2e",
11625            wait,
11626            |state, budget| async move {
11627                run_e2e_with_retry(
11628                    state,
11629                    &shell,
11630                    &commands,
11631                    &winner.worktree,
11632                    budget,
11633                    "current head e2e",
11634                )
11635                .await
11636            },
11637        )
11638        .await?;
11639        outcomes
11640    };
11641    let unresolved = verify_inconclusive(&outcomes);
11642    let passed = !unresolved
11643        && outcomes.iter().all(CommandOutcome::ok)
11644        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11645    state.head_e2e = Some(crate::run::GateRepairE2e {
11646        head: head.clone(),
11647        passed,
11648        outcomes,
11649    });
11650    if !passed && !unresolved {
11651        state.status = RunStatus::Blocked;
11652        state.event(
11653            "e2e",
11654            format!("verify.e2e failed on head {head}; nothing published"),
11655        );
11656    }
11657    state.save()?;
11658    Ok(passed)
11659}
11660
11661/// Recheck a repair's e2e requirement across failures, resource deferrals and
11662/// process interruptions. A successful result only applies to its exact SHA.
11663pub(crate) async fn check_gate_repair_e2e(
11664    state: &mut RunState,
11665    winner: &Candidate,
11666    wait: &crate::repo_lock::WaitPolicy,
11667) -> Result<GateFix> {
11668    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11669        return Ok(GateFix::Retry);
11670    }
11671    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11672    if state
11673        .gate_repair_e2e
11674        .as_ref()
11675        .is_some_and(|r| r.head == head && r.passed)
11676    {
11677        return Ok(GateFix::Retry);
11678    }
11679    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11680        head: head.clone(),
11681        passed: false,
11682        outcomes: Vec::new(),
11683    });
11684    state.save()?;
11685    let commands = state.config.verify.e2e.clone();
11686    let outcomes = if commands.is_empty() {
11687        Vec::new()
11688    } else {
11689        let shell = state.config.shell();
11690        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11691        let cache_dir = state.config.cache_dir();
11692        let (outcomes, _) = with_cache_lease(
11693            state,
11694            cache_dir.as_deref(),
11695            "e2e",
11696            "e2e",
11697            &winner.worktree,
11698            &head,
11699            timeout,
11700            "gate repair e2e",
11701            wait,
11702            |state, budget| async move {
11703                run_e2e_with_retry(
11704                    state,
11705                    &shell,
11706                    &commands,
11707                    &winner.worktree,
11708                    budget,
11709                    "gate repair e2e",
11710                )
11711                .await
11712            },
11713        )
11714        .await?;
11715        outcomes
11716    };
11717    let deferred = verify_inconclusive(&outcomes);
11718    let passed = outcomes.iter().all(CommandOutcome::ok)
11719        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11720    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11721        head,
11722        passed,
11723        outcomes,
11724    });
11725    state.save()?;
11726    if deferred {
11727        Ok(GateFix::Defer)
11728    } else if !passed {
11729        state.event(
11730            "gate",
11731            "verify.e2e failed after gate repair; nothing published",
11732        );
11733        Ok(GateFix::Stop)
11734    } else {
11735        Ok(GateFix::Retry)
11736    }
11737}
11738
11739/// Run the same configured gate for a landing head and a fresh graph.
11740pub(crate) async fn run_landing_gate(
11741    state: &mut RunState,
11742    winner: &Candidate,
11743    wait: &crate::repo_lock::WaitPolicy,
11744) -> Result<Vec<CommandOutcome>> {
11745    let shell = state.config.shell();
11746    let gate_commands = state.config.verify.gate.clone();
11747    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11748    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11749        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
11750        Vec::new()
11751    } else {
11752        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11753        let cache_dir = state.config.cache_dir();
11754        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11755        let (outcomes, _) = with_cache_lease(
11756            state,
11757            cache_dir.as_deref(),
11758            "gate",
11759            "gate",
11760            &winner.worktree,
11761            &head,
11762            timeout,
11763            "final gate",
11764            wait,
11765            |state, budget| {
11766                let shell = shell.clone();
11767                let gate_commands = gate_commands.clone();
11768                let winner = winner.clone();
11769                async move {
11770                    Runner::run_pre_gate(state, &winner).await;
11771                    // Bind command evidence after mechanical fixes, before commands start.
11772                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
11773                    let (outcomes, timed_out_pids) = run_commands(
11774                        state,
11775                        "gate",
11776                        "gate",
11777                        0,
11778                        &shell,
11779                        &gate_commands,
11780                        &winner.worktree,
11781                        budget,
11782                    )
11783                    .await;
11784                    (outcomes, false, timed_out_pids)
11785                }
11786            },
11787        )
11788        .await?;
11789        outcomes
11790    };
11791    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
11792    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
11793        let mut outcomes = outcomes;
11794        outcomes.push(CommandOutcome {
11795            command: "gate head stability".into(), code: None,
11796            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
11797            duration_ms: 0, resource_blocked: false,
11798        });
11799        return Ok(outcomes);
11800    }
11801    if outcomes.is_empty() {
11802        // Nothing configured to check — distinct from every other
11803        // silence in this run's event log, since an empty `gate` alone
11804        // no longer says whether the gate ran at all (see
11805        // `RunState::gate_ran`'s own doc).
11806        state.event("gate", "no gate commands configured; not tested");
11807    }
11808    for o in &outcomes {
11809        state.event(
11810            "gate",
11811            format!(
11812                "`{}` -> {}",
11813                o.command,
11814                if o.ok() {
11815                    "pass".to_owned()
11816                } else {
11817                    format!(
11818                        "FAIL ({:?})\n{}",
11819                        o.code,
11820                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11821                    )
11822                }
11823            ),
11824        );
11825    }
11826    Ok(outcomes)
11827}
11828
11829/// The pull request title to hand to `land::merge_subject`. A review-only run
11830/// opened by an earlier build titled its pull request with the review prompt;
11831/// that title is dropped (empty, so the fallback applies) rather than landed.
11832/// Any other title, including an operator's rename, passes through untouched,
11833/// and so does every title of a run that implements a task.
11834pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11835    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11836        ""
11837    } else {
11838        pr_title
11839    }
11840}
11841
11842/// What the squash subject falls back to when the pull request title is empty
11843/// or candidate-shaped: for a review-only run the derived title, never the
11844/// review prompt held in `instruction`.
11845pub fn landing_subject_source(state: &RunState) -> String {
11846    if is_review_run(state) {
11847        let winner = state.candidates.first().map_or('A', |c| c.label);
11848        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11849    }
11850    state.instruction.clone()
11851}
11852
11853/// `summary` without its `TITLE:` line, which the pull request title already
11854/// carries.
11855fn summary_without_title(summary: &str) -> String {
11856    let mut lines = summary.trim().lines().peekable();
11857    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11858        lines.next();
11859    }
11860    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11861}
11862
11863/// The pull request title and body for the winning candidate.
11864///
11865/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
11866/// the task's own opening line via [`queue::title_from`] when there is none.
11867/// `state.instruction` can open with blank lines (`task_text` only rejects a
11868/// body that is blank *entirely*), which `title_from` skips.
11869///
11870/// Publishing uses an independent description generated from the final diff
11871/// and structured evidence. Internal summaries and reports remain local.
11872/// Non-publishing previews may display the original task and candidate summary.
11873/// The footer retains the run and candidate provenance.
11874#[cfg(test)]
11875fn pr_message(state: &RunState, winner: char) -> PrMessage {
11876    pr_message_with(state, winner, None)
11877}
11878
11879/// [`pr_message`] with what the branch of a review-only run says about itself.
11880/// `facts` is ignored for a run that implements a task.
11881#[cfg(test)]
11882fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11883    let raw = pr_message_raw(
11884        state,
11885        winner,
11886        facts,
11887        crate::pr_outcome::FinalHead::Unreported,
11888        ChangeNote::NotAsked,
11889    );
11890    let id = crate::scrub::Identity::current();
11891    PrMessage {
11892        title: crate::scrub::scrub(&raw.title, &id),
11893        body: crate::scrub::scrub(&raw.body, &id),
11894    }
11895}
11896
11897/// What the public description may say about the change itself.
11898#[derive(Debug, Clone, Copy)]
11899enum ChangeNote<'a> {
11900    /// A local preview or non-publishing merge mode: no public generation needed.
11901    NotAsked,
11902    /// A summary written from the final diff.
11903    Summary(&'a crate::run::ChangeSummary),
11904    /// A summary was asked for and could not be produced; the text says why.
11905    Unavailable(&'a str),
11906}
11907
11908fn pr_message_raw(
11909    state: &RunState,
11910    winner: char,
11911    facts: Option<&BranchFacts>,
11912    head: crate::pr_outcome::FinalHead<'_>,
11913    change: ChangeNote<'_>,
11914) -> PrMessage {
11915    if !matches!(change, ChangeNote::NotAsked) {
11916        let (title, mut body) = match change {
11917            ChangeNote::Summary(c) => (
11918                c.title.clone(),
11919                format!("## Summary\n\n{}\n\n", c.body.trim()),
11920            ),
11921            ChangeNote::Unavailable(why) => (
11922                neutral_title(state, winner),
11923                format!(
11924                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11925                ),
11926            ),
11927            ChangeNote::NotAsked => unreachable!(),
11928        };
11929        body.push_str(&public_pr_evidence(state, head));
11930        body.push_str(&format!(
11931            "\n---\nmagi:run/{} magi:candidate-{}\n",
11932            state.id,
11933            winner.to_ascii_lowercase()
11934        ));
11935        return PrMessage { title, body };
11936    }
11937    pr_preview(state, winner, facts, head)
11938}
11939
11940/// Append public facts from the run records after every prose generation,
11941/// including posting-gate rewrites. The agent cannot drop this provenance.
11942fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11943    let mut body = crate::pr_outcome::review_rounds(state);
11944    body.push_str(&crate::pr_outcome::final_verification(state, head));
11945    let open = state.last_round_findings();
11946    if !open.is_empty() {
11947        body.push_str("## Open review findings\n\n");
11948        for f in open {
11949            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11950        }
11951        body.push('\n');
11952    }
11953    if let Some(c) = state.change_summary.as_ref()
11954        && head == crate::pr_outcome::FinalHead::At(&c.head)
11955    {
11956        body.push_str(&format!(
11957            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11958            c.base, c.head
11959        ));
11960    }
11961    body
11962}
11963
11964/// Local-only preview of the operator's task and candidate summary.
11965fn pr_preview(
11966    state: &RunState,
11967    winner: char,
11968    facts: Option<&BranchFacts>,
11969    head: crate::pr_outcome::FinalHead<'_>,
11970) -> PrMessage {
11971    let summary = state
11972        .candidates
11973        .iter()
11974        .find(|c| c.label == winner)
11975        .map(|c| c.summary.as_str())
11976        .unwrap_or_default();
11977    // The fallback is the operator's own words and may not be English; GitHub
11978    // text always is, so a non-English task gets a neutral title instead.
11979    let review = is_review_run(state);
11980    let title = if review {
11981        facts
11982            .and_then(|f| {
11983                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11984            })
11985            .or_else(|| review_title(state))
11986            .or_else(|| {
11987                state
11988                    .candidates
11989                    .iter()
11990                    .find(|c| c.label == winner)
11991                    .filter(|c| !c.branch.starts_with("magi/"))
11992                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11993            })
11994            .unwrap_or_else(|| neutral_title(state, winner))
11995    } else {
11996        summary_title(summary).unwrap_or_else(|| {
11997            english_title(
11998                &state.instruction,
11999                PR_TITLE_MAX,
12000                state.config.graph.github_english(),
12001            )
12002            .unwrap_or_else(|| neutral_title(state, winner))
12003        })
12004    };
12005
12006    let mut body = String::new();
12007    let what = summary_without_title(summary);
12008    if !what.is_empty() {
12009        body.push_str("## Summary\n\n");
12010        body.push_str(&what);
12011        body.push_str("\n\n");
12012    }
12013
12014    // Review history uses structured facts across all rounds, never reports.
12015    body.push_str(&crate::pr_outcome::review_rounds(state));
12016    body.push_str(&crate::pr_outcome::final_verification(state, head));
12017    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12018
12019    let open = state.open_findings();
12020    if !open.is_empty() {
12021        body.push_str("## Open review findings\n\n");
12022        for f in &open {
12023            body.push_str(&format!(
12024                "- `{}` [{:?}] {}\n",
12025                f.id, f.severity, "Context retained in local review records."
12026            ));
12027        }
12028        body.push('\n');
12029    }
12030
12031    if let Some(fix) = fix
12032        && !fix.rejected.is_empty()
12033    {
12034        body.push_str("## Declined by the fixer\n\n");
12035        for r in &fix.rejected {
12036            body.push_str(&format!(
12037                "- `{}`: {}\n",
12038                r.id, "Reason retained in local review records."
12039            ));
12040        }
12041        body.push('\n');
12042    }
12043
12044    if review {
12045        // The review prompt is not the task; list what the branch carries.
12046        body.push_str("## Commits under review\n\n");
12047        if let Some(facts) = facts {
12048            let mut left = COMMITS_MAX_CHARS;
12049            for (i, (subject, text)) in facts.commits.iter().enumerate() {
12050                let mut entry = format!("- {}\n", markdown_safe(subject));
12051                for l in markdown_safe(text).lines() {
12052                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
12053                }
12054                if left == 0 {
12055                    body.push_str(&format!(
12056                        "- ... {} more commit(s)\n",
12057                        facts.commits.len() - i
12058                    ));
12059                    break;
12060                }
12061                if entry.len() > left {
12062                    // Even the first commit is cut: one huge body must not
12063                    // push the whole description past GitHub's limit.
12064                    let mut end = left;
12065                    while !entry.is_char_boundary(end) {
12066                        end -= 1;
12067                    }
12068                    entry.truncate(end);
12069                    entry.push_str("\n  ... (truncated)\n");
12070                    left = 0;
12071                } else {
12072                    left -= entry.len();
12073                }
12074                body.push_str(&entry);
12075            }
12076            if !facts.stat.trim().is_empty() {
12077                body.push_str(&format!(
12078                    "\n## Diff stat\n\n```\n{}\n```\n",
12079                    markdown_safe(facts.stat.trim())
12080                ));
12081            }
12082        } else {
12083            match &state.reviewed_commits {
12084                Some(subjects) => {
12085                    for s in subjects {
12086                        body.push_str(&format!("- {}\n", s.trim()));
12087                    }
12088                }
12089                None => {
12090                    // An older run kept only the prompt, with the commit list
12091                    // after its first paragraph.
12092                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
12093                    body.push_str(rest.trim());
12094                    body.push('\n');
12095                }
12096            }
12097        }
12098    } else {
12099        let task = state.instruction.trim();
12100        let task = if task.is_empty() {
12101            "(empty task)"
12102        } else {
12103            task
12104        };
12105        body.push_str(&format!(
12106            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
12107            task.replace("</details>", "&lt;/details&gt;")
12108        ));
12109    }
12110
12111    body.push_str(&format!(
12112        "\n---\nmagi:run/{} magi:candidate-{}\n",
12113        state.id,
12114        winner.to_ascii_lowercase()
12115    ));
12116
12117    PrMessage { title, body }
12118}
12119
12120/// Source imports have already been prepared from structured inputs. Legacy
12121/// inferred references must never instruct a seat to reconstruct those imports.
12122fn seeded_instruction(state: &RunState) -> String {
12123    let mut inputs = Vec::new();
12124    if let Some(start) = &state.sources.start_from {
12125        inputs.push(format!(
12126            "Start from `{}` at `{}` (including ancestry).",
12127            start.token, start.sha
12128        ));
12129    }
12130    for pick in &state.sources.pick_commits {
12131        inputs.push(format!(
12132            "Pick `{}` at `{}` (single commit).",
12133            pick.token, pick.sha
12134        ));
12135    }
12136    if inputs.is_empty() {
12137        state.instruction.clone()
12138    } else {
12139        format!(
12140            "{}\n\n# Explicit source inputs already prepared\n\n{}",
12141            state.instruction,
12142            inputs.join("\n")
12143        )
12144    }
12145}
12146
12147/// Does the winner have no commits ahead of the base it would land on?
12148/// Any failure to find out reads as "not empty": the merge then behaves as it
12149/// always did rather than refusing on a guess.
12150async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
12151    let base = &state.base_branch;
12152    let mut against = base.clone();
12153    if mode == MergeMode::Pr {
12154        // A pull request lands on the remote's base, never the local branch
12155        // of the same name: if that cannot be read, "not empty" is the safe
12156        // answer.
12157        let remote = &state.config.merge.remote;
12158        let tracking = format!("{remote}/{base}");
12159        let fetched = git::fetch(repo, remote, base).await;
12160        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
12161            against = tracking;
12162        } else {
12163            return false;
12164        }
12165    }
12166    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
12167}
12168
12169/// Why nothing was opened for an empty winner, with what the task's own
12170/// references resolved to.
12171fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
12172    let mut detail = format!(
12173        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
12174         nothing to open a pull request for"
12175    );
12176    detail.push_str(
12177        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
12178    );
12179    detail
12180}
12181
12182/// What the `Pr` merge does once it knows whether the branch already has an
12183/// open pull request.
12184#[derive(Debug, PartialEq, Eq)]
12185enum PrPlan {
12186    Create,
12187    Adopt { url: String, title: String },
12188    Stop(String),
12189}
12190
12191/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
12192/// a failed lookup -> stop with the real reason. Never guesses.
12193fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
12194    match found {
12195        Ok(land::OpenPr::None) => PrPlan::Create,
12196        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
12197        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
12198            "several open pull requests exist for this branch, not picking one: {}",
12199            urls.join(" ")
12200        )),
12201        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
12202    }
12203}
12204
12205/// `gh pr create`, returning the PR url.
12206pub(crate) async fn gh_pr_create(
12207    cwd: &Path,
12208    base: &str,
12209    head: &str,
12210    title: &str,
12211    body: &str,
12212) -> Result<String> {
12213    let out = tokio::process::Command::new("gh")
12214        .args([
12215            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
12216        ])
12217        .current_dir(cwd)
12218        .quiet()
12219        .stdin(std::process::Stdio::null())
12220        .output()
12221        .await
12222        .context("spawn gh")?;
12223    if out.status.success() {
12224        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
12225    } else {
12226        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
12227    }
12228}
12229
12230/// Never force deletion of uncommitted work, including unknown legacy trees.
12231async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
12232    match std::fs::symlink_metadata(path) {
12233        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
12234            // A previous cleanup may have removed the tree before a branch
12235            // deletion or state save failed. Unregister it if still present.
12236            git::worktree_remove_clean(repo, path).await?;
12237            return Ok(());
12238        }
12239        Err(e) => return Err(e.into()),
12240        Ok(_) => {}
12241    }
12242    if !git::is_clean(path).await? {
12243        bail!(
12244            "worktree has uncommitted work; refusing removal: {}",
12245            path.display()
12246        );
12247    }
12248    if !git::worktree_remove_clean(repo, path).await? {
12249        bail!("worktree removal refused: {}", path.display());
12250    }
12251    Ok(())
12252}
12253
12254/// Tear a run's worktrees and branches down.
12255///
12256/// `home` is where the updated `run.json` is saved (via
12257/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
12258/// a housekeeping pass already has its own honest `home` handed to it, and
12259/// falling through to the global here would write back through whichever
12260/// directory some other process or test pinned into that `OnceLock` first,
12261/// not the one the caller actually resolved its `runs` and `state` from.
12262pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
12263    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
12264        bail!("refusing fold: {reason}");
12265    }
12266    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
12267        bail!("candidate preservation incomplete; refusing cleanup");
12268    }
12269    // Serialize ordinary folding with the same starts/resumes as target-only
12270    // eviction. Otherwise a second janitor could remove the whole worktree
12271    // around a live output borrower even though target eviction was fenced.
12272    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
12273    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
12274    let _driver = crate::driver::maintenance(home, &state.id)?;
12275    let mut output_guards = Vec::new();
12276    for entry in std::fs::read_dir(state.worktree_root())
12277        .into_iter()
12278        .flatten()
12279        .flatten()
12280    {
12281        if entry.path().join(".git").is_file() {
12282            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
12283        }
12284    }
12285    let repo = state.repo.clone();
12286    // CLI, Web and automatic cleanup all enter here, independently of
12287    // Runner::fold_losers. Protect deletion and shared config release too.
12288    let wait = repository_wait(state, None);
12289    let _coordination = match coordinate_operation(state, &repo, &wait).await {
12290        Ok(lease) => lease,
12291        Err(e) => {
12292            state.save_under(home)?;
12293            return Err(e);
12294        }
12295    };
12296    // Confirmed merges dispose of their owned sources after durable evidence
12297    // capture. Preflight every source before removing anything.
12298    let _source_outputs = if state.status == RunStatus::Merged
12299        && state.merge.as_ref().is_some_and(|merge| merge.ok)
12300    {
12301        verify_merged_fold_sources(state, drop_winner, home).await?
12302    } else {
12303        Vec::new()
12304    };
12305    let root = state.worktree_root();
12306    let winner = state.tally.as_ref().map(|t| t.winner);
12307    let mut removed = Vec::new();
12308
12309    for i in 0..state.candidates.len() {
12310        let c = state.candidates[i].clone();
12311        let is_winner = Some(c.label) == winner;
12312        if is_winner && !drop_winner {
12313            continue;
12314        }
12315        let existed = c.worktree.exists();
12316        // A vanished directory can still have a Git registration that keeps
12317        // its branch checked out. Remove that registration before the branch.
12318        remove_preserved_worktree(&repo, &c.worktree).await?;
12319        if existed {
12320            removed.push(c.worktree.to_string_lossy().into_owned());
12321        }
12322        // A branch handed to a later run (and its pull request) is not this
12323        // run's to delete.
12324        let handed_over = state.released_branches.contains(&c.branch);
12325        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12326            if !git::branch_delete(&repo, &c.branch).await? {
12327                bail!("candidate branch removal refused");
12328            }
12329            removed.push(c.branch.clone());
12330        }
12331        state.candidates[i].folded = true;
12332    }
12333
12334    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12335        let path = name.path();
12336        let keep = !drop_winner
12337            && winner.is_some_and(|w| {
12338                path.file_name()
12339                    .is_some_and(|n| n == format!("cand-{w}").as_str())
12340            });
12341        if keep {
12342            continue;
12343        }
12344        remove_preserved_worktree(&repo, &path).await?;
12345        removed.push(path.to_string_lossy().into_owned());
12346    }
12347
12348    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
12349    // judge worktrees, so once the loop above has cleared all of them out,
12350    // the parent is a bare directory nobody else was ever going to remove -
12351    // git only ever managed what was inside it. Left alone, one of these
12352    // accumulates per fully-folded run; the operator's own machine had 74.
12353    // `remove_if_empty` re-checks rather than assuming: a run whose winner
12354    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
12355    // so does anything a run never claimed that happens to share the bay.
12356    remove_if_empty(&root);
12357
12358    if state.enabled_worktree_config && drop_winner {
12359        // A release, not a raw disable: some sibling run in this repository
12360        // may still hold its own reference (see `git::acquire_worktree_config`),
12361        // and only the last release actually turns the setting back off.
12362        git::release_worktree_config(&repo).await.ok();
12363        state.enabled_worktree_config = false;
12364    }
12365    state.save_under(home)?;
12366    Ok(removed)
12367}
12368
12369async fn verify_merged_fold_sources(
12370    state: &RunState,
12371    drop_winner: bool,
12372    home: &Path,
12373) -> Result<Vec<std::fs::File>> {
12374    let winner = state.tally.as_ref().map(|t| t.winner);
12375    let mut heads = BTreeSet::new();
12376    for candidate in &state.candidates {
12377        if Some(candidate.label) == winner && !drop_winner {
12378            continue;
12379        }
12380        anyhow::ensure!(
12381            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12382            "candidate recovery source is outside the inventoried run root: {}",
12383            candidate.worktree.display()
12384        );
12385        match std::fs::symlink_metadata(&candidate.worktree) {
12386            Ok(_) => {}
12387            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12388            Err(error) => return Err(error.into()),
12389        }
12390        if !state.released_branches.contains(&candidate.branch)
12391            && git::branch_exists(&state.repo, &candidate.branch).await?
12392        {
12393            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12394        }
12395    }
12396    let mut cargo_guards = Vec::new();
12397    let entries = match std::fs::read_dir(state.worktree_root()) {
12398        Ok(entries) => Some(entries),
12399        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12400        Err(error) => return Err(error.into()),
12401    };
12402    for entry in entries.into_iter().flatten() {
12403        let path = entry?.path();
12404        let metadata = std::fs::symlink_metadata(&path)?;
12405        anyhow::ensure!(
12406            metadata.is_dir() && !metadata.file_type().is_symlink(),
12407            "non-directory or symlink recovery source retained: {}",
12408            path.display()
12409        );
12410        if !drop_winner
12411            && winner.is_some_and(|w| {
12412                path.file_name()
12413                    .is_some_and(|name| name == format!("cand-{w}").as_str())
12414            })
12415        {
12416            continue;
12417        }
12418        anyhow::ensure!(
12419            path.join(".git").is_file(),
12420            "unregistered recovery source retained: {}",
12421            path.display()
12422        );
12423        anyhow::ensure!(
12424            git::git(
12425                &path,
12426                &[
12427                    "status",
12428                    "--porcelain",
12429                    "--untracked-files=all",
12430                    "--ignored",
12431                    "--",
12432                    ".",
12433                    ":(exclude)target"
12434                ]
12435            )
12436            .await?
12437            .is_empty(),
12438            "source changes or ignored recovery files retained: {}",
12439            path.display()
12440        );
12441        match std::fs::symlink_metadata(path.join("target")) {
12442            Ok(_) => {
12443                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
12444            }
12445            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12446            Err(error) => return Err(error.into()),
12447        }
12448        for checkpoint in [
12449            "rebase-merge",
12450            "rebase-apply",
12451            "sequencer",
12452            "CHERRY_PICK_HEAD",
12453            "MERGE_HEAD",
12454            "REBASE_HEAD",
12455            "REVERT_HEAD",
12456            "BISECT_LOG",
12457            "AUTO_MERGE",
12458            "MERGE_MSG",
12459        ] {
12460            let location = git::git(
12461                &path,
12462                &[
12463                    "rev-parse",
12464                    "--path-format=absolute",
12465                    "--git-path",
12466                    checkpoint,
12467                ],
12468            )
12469            .await?;
12470            anyhow::ensure!(
12471                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
12472                if e.kind() == std::io::ErrorKind::NotFound),
12473                "Git recovery checkpoint retained: {checkpoint}"
12474            );
12475        }
12476        heads.insert(git::rev_parse(&path, "HEAD").await?);
12477    }
12478    crate::fold_evidence::preserve(state, home, &heads).await?;
12479
12480    Ok(cargo_guards)
12481}
12482
12483/// Remove `dir` if it exists and has nothing in it.
12484///
12485/// Best-effort and silent by design: a directory that is not empty (a run
12486/// whose winner is still parked there, a stray file some other process left)
12487/// is exactly the case this must refuse, and a directory that is already gone
12488/// is not a failure worth reporting either. `std::fs::remove_dir` itself
12489/// already refuses a non-empty directory, so the emptiness check below is
12490/// belt, not suspenders - it is what keeps this from ever attempting the
12491/// removal in the case that matters, rather than trusting `remove_dir`'s
12492/// error path to have no side effects if it ever changed.
12493fn remove_if_empty(dir: &Path) {
12494    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
12495        std::fs::remove_dir(dir).ok();
12496    }
12497}
12498
12499/// Severity of the worst open finding in the last review round, for reporting.
12500pub fn worst_open(state: &RunState) -> Option<Severity> {
12501    state
12502        .reviews
12503        .last()?
12504        .reviews
12505        .iter()
12506        .flat_map(|r| r.findings.iter())
12507        .map(|f| f.severity)
12508        .max()
12509}
12510
12511/// `[worktree] setup` hides its products per worktree, which needs
12512/// `extensions.worktreeConfig`. Taken once per run and released with the run's
12513/// worktrees, like the hook's; every place that runs setup calls this first.
12514async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12515    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12516        git::acquire_worktree_config(repo).await?;
12517        state.enabled_worktree_config = true;
12518    }
12519    Ok(())
12520}
12521
12522#[cfg(test)]
12523mod tests {
12524    #[tokio::test]
12525    async fn launcher_base_fetch_requires_remote_success() {
12526        let tmp = tempfile::tempdir().unwrap();
12527        let repo = tmp.path().join("repo");
12528        let remote = tmp.path().join("remote");
12529        std::fs::create_dir(&repo).unwrap();
12530        std::fs::create_dir(&remote).unwrap();
12531        git::git(&remote, &["init", "--bare", "-b", "main"])
12532            .await
12533            .unwrap();
12534        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12535        git::git(
12536            &repo,
12537            &[
12538                "-c",
12539                "user.name=Fixture",
12540                "-c",
12541                "user.email=fixture@example.invalid",
12542                "commit",
12543                "--allow-empty",
12544                "-m",
12545                "fixture",
12546            ],
12547        )
12548        .await
12549        .unwrap();
12550        git::git(
12551            &repo,
12552            &["remote", "add", "origin", remote.to_str().unwrap()],
12553        )
12554        .await
12555        .unwrap();
12556        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12557        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12558        assert_eq!(
12559            resolve_base(&repo, "main", "origin", None)
12560                .await
12561                .unwrap()
12562                .commit,
12563            head
12564        );
12565        #[cfg(unix)]
12566        {
12567            let policy: crate::config::Coordination = toml::from_str(
12568                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12569            ).unwrap();
12570            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12571            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12572            let held = crate::repo_lock::SeatLease::operation_with(
12573                &repo,
12574                Some(&policy),
12575                &crate::repo_lock::WaitPolicy::default(),
12576            )
12577            .await
12578            .unwrap();
12579            assert!(held.is_some());
12580            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12581            tokio::pin!(fetch);
12582            assert!(
12583                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12584                    .await
12585                    .is_err()
12586            );
12587            assert!(
12588                !repo.join(".git/FETCH_HEAD").exists(),
12589                "no fetch before admission"
12590            );
12591            drop(held);
12592            let fetched = fetch.await.unwrap();
12593            assert_eq!(fetched.commit, head);
12594            assert_eq!(fetched.remote, "origin");
12595            assert_eq!(fetched.branch, "main");
12596        }
12597        // A surviving tracking ref is not successful fetch evidence.
12598        git::git(
12599            &repo,
12600            &[
12601                "remote",
12602                "set-url",
12603                "origin",
12604                tmp.path().join("missing").to_str().unwrap(),
12605            ],
12606        )
12607        .await
12608        .unwrap();
12609        let error = resolve_base(&repo, "main", "origin", None)
12610            .await
12611            .unwrap_err()
12612            .to_string();
12613        assert!(error.contains("cannot read origin/main"));
12614        assert!(error.contains("refusing to branch off"));
12615    }
12616
12617    #[test]
12618    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12619        let leaked = vec!["chore(deps): update a crate".to_owned()];
12620        let own = "fix(daemon): apply a chosen action";
12621        assert!(should_retitle("", own, &leaked));
12622        assert!(should_retitle(
12623            &format!("{REVIEW_PROMPT_OPENING} `x`"),
12624            own,
12625            &leaked
12626        ));
12627        assert!(should_retitle(
12628            "chore: land candidate A of run 1",
12629            own,
12630            &leaked
12631        ));
12632        assert!(should_retitle(
12633            "magi: candidate A (uncommitted work)",
12634            own,
12635            &leaked
12636        ));
12637        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12638        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12639        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12640    }
12641
12642    #[test]
12643    fn pr_merge_plan_creates_adopts_or_stops() {
12644        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12645        assert_eq!(
12646            pr_merge_plan(Ok(land::OpenPr::One {
12647                url: "u".into(),
12648                title: "t".into()
12649            })),
12650            PrPlan::Adopt {
12651                url: "u".into(),
12652                title: "t".into()
12653            }
12654        );
12655        let PrPlan::Stop(many) =
12656            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12657        else {
12658            panic!("many must stop");
12659        };
12660        assert!(many.contains('a') && many.contains('b'));
12661        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12662            panic!("a failed lookup must stop");
12663        };
12664        assert!(err.contains("bad token"));
12665    }
12666
12667    use super::*;
12668    use crate::run::GateStatus;
12669    use std::collections::BTreeMap;
12670    use std::time::Duration;
12671
12672    fn conductor() -> AgentSpec {
12673        AgentSpec {
12674            id: "conductor".to_owned(),
12675            kind: crate::config::AgentKind::Command,
12676            model: None,
12677            command: vec!["true".to_owned()],
12678            extra_args: Vec::new(),
12679            env: BTreeMap::new(),
12680            prompt_delivery: None,
12681            max_turn_secs: None,
12682        }
12683    }
12684
12685    fn spec(id: &str) -> AgentSpec {
12686        AgentSpec {
12687            id: id.to_owned(),
12688            kind: crate::config::AgentKind::Command,
12689            model: None,
12690            command: vec!["true".to_owned()],
12691            extra_args: Vec::new(),
12692            env: BTreeMap::new(),
12693            prompt_delivery: None,
12694            max_turn_secs: None,
12695        }
12696    }
12697
12698    fn ids(xs: &[&str]) -> BTreeSet<String> {
12699        xs.iter().map(|s| (*s).to_owned()).collect()
12700    }
12701
12702    #[test]
12703    fn pick_successor_skips_an_agent_another_seat_holds() {
12704        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
12705        let roster = [spec("a"), spec("b"), spec("c")];
12706        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12707        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12708    }
12709
12710    #[test]
12711    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12712        // The other seat started on c but was handed to d; c is free again.
12713        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12714        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12715        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12716        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12717        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12718    }
12719
12720    #[test]
12721    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12722        let roster = [spec("a"), spec("b")];
12723        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12724        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12725        let carried = ids(&["b"]);
12726        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12727        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12728    }
12729
12730    #[test]
12731    fn pick_successor_returns_none_without_any_untried_successor() {
12732        let roster = [spec("a"), spec("b")];
12733        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12734        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12735        let carried = ids(&["a"]);
12736        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12737    }
12738
12739    #[test]
12740    fn pick_successor_rescue_avoids_another_seats_occupant() {
12741        // b is a carried failure and free; a is held by the other seat.
12742        let roster = [spec("a"), spec("b"), spec("c")];
12743        let carried = ids(&["b", "c"]);
12744        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12745        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12746    }
12747
12748    #[test]
12749    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12750        let roster = [spec("a"), spec("b"), spec("c")];
12751        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12752        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12753    }
12754
12755    #[test]
12756    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12757        let roster = [spec("a"), spec("b"), spec("c")];
12758        let failed = ids(&["a", "b", "c"]);
12759        // Rescue looks at the whole roster, once per id, then runs out.
12760        let mut tried = ids(&["b"]);
12761        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12762        assert_eq!(first.id, "a");
12763        tried.insert(first.id.clone());
12764        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12765        assert_eq!(second.id, "c");
12766        tried.insert(second.id.clone());
12767        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12768    }
12769
12770    #[test]
12771    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12772        let roster = [spec("a"), spec("b")];
12773        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12774        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12775    }
12776
12777    #[test]
12778    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12779        let roster = [spec("a"), spec("b"), spec("c")];
12780        let h = SeatHistory {
12781            failed: ids(&["a"]),
12782            last_ok: Some("b".to_owned()),
12783            last_fail: None,
12784        };
12785        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12786        // A last answerer that left the roster, or later failed, is ignored.
12787        let gone = SeatHistory {
12788            last_ok: Some("zzz".to_owned()),
12789            ..h.clone()
12790        };
12791        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12792        let failed = SeatHistory {
12793            failed: ids(&["a", "b"]),
12794            last_ok: Some("b".to_owned()),
12795            last_fail: None,
12796        };
12797        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12798    }
12799
12800    #[test]
12801    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12802        let first = SeatState::new("review-1", "alpha", 7);
12803        let next = handover_seat("review-1", "gamma", 7);
12804        assert_ne!(first.claude_session, next.claude_session);
12805    }
12806
12807    #[test]
12808    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12809        let mut state = state_with_summary("x", "y");
12810        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12811        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12812        assert_ne!(a.claude_session, b.claude_session);
12813    }
12814
12815    #[test]
12816    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12817        let roster = [spec("a"), spec("b")];
12818        let h = SeatHistory {
12819            failed: ids(&["a", "b"]),
12820            ..SeatHistory::default()
12821        };
12822        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12823        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12824        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12825    }
12826
12827    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12828    // fallback loop depends on to terminate: it must walk forward from the
12829    // seat's own position, never restart at the front of the roster, and it
12830    // must never hand back an id already tried, however many times that id
12831    // happens to appear.
12832
12833    #[test]
12834    fn failure_signature_ignores_numbers_and_paths() {
12835        assert_eq!(
12836            failure_signature("exited with Some(2) and no usable output"),
12837            failure_signature("exited with Some(137) and no usable output")
12838        );
12839        assert_eq!(
12840            failure_signature("process exited with no usable output (exit code 2)"),
12841            failure_signature("process exited with no usable output (exit code 137)")
12842        );
12843        for (old, new) in [
12844            (
12845                "exited with Some(2) and no usable output",
12846                "process exited with no usable output (exit code 2)",
12847            ),
12848            (
12849                "exited with None and no usable output",
12850                "process exited with no usable output (exit code unavailable)",
12851            ),
12852        ] {
12853            let previous = FailClass::Other(failure_signature(old));
12854            let current = FailClass::Other(failure_signature(new));
12855            assert_eq!(previous, current, "resumption retains the failure class");
12856            assert!(!should_hand_over(Some(&previous), &current));
12857        }
12858        assert_eq!(
12859            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12860            failure_signature("cannot open /var/x.txt: denied")
12861        );
12862        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12863    }
12864
12865    #[test]
12866    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12867        let other = FailClass::Other("x".into());
12868        assert!(should_hand_over(None, &FailClass::Quota));
12869        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12870        assert!(should_hand_over(
12871            Some(&FailClass::Timeout),
12872            &FailClass::Timeout
12873        ));
12874        assert!(should_hand_over(None, &other));
12875        assert!(!should_hand_over(Some(&other), &other));
12876        assert!(should_hand_over(
12877            Some(&other),
12878            &FailClass::Other("y".into())
12879        ));
12880        // A quota or timeout in between ends the run of identical failures.
12881        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12882        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12883    }
12884
12885    #[test]
12886    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12887        let a = SeatState::new("judge-1", "alpha", 7);
12888        let b = handover_seat("judge-1", "beta", 7);
12889        assert_ne!(a.claude_session, b.claude_session);
12890        assert_eq!(b.turns, 0);
12891    }
12892
12893    #[test]
12894    fn a_timeout_is_classified_apart_from_other_failures() {
12895        assert_eq!(
12896            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12897            Some(FailClass::Timeout)
12898        );
12899        assert!(matches!(
12900            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12901            Some(FailClass::Other(_))
12902        ));
12903    }
12904
12905    #[test]
12906    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12907        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12908        let tried = BTreeSet::from(["beta".to_owned()]);
12909        // beta sits at index 1; the next candidate is gamma, never alpha —
12910        // which is very likely a different candidate slot's own agent.
12911        let next = next_untried_in_roster(&roster, 1, &tried);
12912        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12913    }
12914
12915    #[test]
12916    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12917        let roster = vec![spec("alpha"), spec("beta")];
12918        let tried = BTreeSet::from(["beta".to_owned()]);
12919        // beta is the roster's last entry: nothing follows it, and alpha —
12920        // earlier in the roster, almost certainly a different candidate
12921        // slot's own agent — must not be reached by wrapping back to it.
12922        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12923    }
12924
12925    #[test]
12926    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12927        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12928        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12929        // beta (index 1) and gamma (index 2, the only entry after it) have
12930        // both been tried; alpha (index 0) never has, but it comes before
12931        // beta's own position, so there is nothing further for this seat.
12932        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12933    }
12934
12935    #[test]
12936    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12937        let roster = vec![spec("a"), spec("a"), spec("b")];
12938        let tried = BTreeSet::from(["a".to_owned()]);
12939        let next = next_untried_in_roster(&roster, 0, &tried);
12940        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12941    }
12942
12943    #[test]
12944    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12945        let roster = vec![spec("a"), spec("b")];
12946        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12947        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12948    }
12949
12950    #[test]
12951    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12952        let dir = tempfile::tempdir().unwrap();
12953        let bay = dir.path().join("ffff");
12954
12955        // Not there yet: nothing to do, nothing to panic on.
12956        remove_if_empty(&bay);
12957        assert!(!bay.exists());
12958
12959        // Something still inside - the winner's worktree, or a stray file -
12960        // keeps the directory standing.
12961        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12962        remove_if_empty(&bay);
12963        assert!(bay.exists(), "non-empty directory must survive");
12964
12965        // Once the last entry is gone, so is the directory itself.
12966        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12967        remove_if_empty(&bay);
12968        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12969    }
12970
12971    // `round_is_clean` is the exact decision this task fixed: a round with a
12972    // seat that never answered must not read the same as a round every seat
12973    // actually reviewed. These are deterministic and process-free by design —
12974    // the equivalent end-to-end check (a real reviewer timing out under a
12975    // live graph run) is a genuine race against wall-clock contention, and a
12976    // spawn slow enough to blow even a generous budget under a loaded test
12977    // run must not turn this specific regression check flaky.
12978
12979    #[test]
12980    fn a_full_panel_that_found_nothing_is_clean() {
12981        assert!(round_is_clean(
12982            0,
12983            true,
12984            2,
12985            2,
12986            0,
12987            IncompleteReviewPolicy::Block
12988        ));
12989    }
12990
12991    #[test]
12992    fn a_missing_seat_is_never_clean_under_the_default_policy() {
12993        assert!(!round_is_clean(
12994            0,
12995            true,
12996            1,
12997            2,
12998            0,
12999            IncompleteReviewPolicy::Block
13000        ));
13001    }
13002
13003    #[test]
13004    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13005        assert!(!round_is_clean(
13006            1,
13007            true,
13008            1,
13009            2,
13010            0,
13011            IncompleteReviewPolicy::Warn
13012        ));
13013    }
13014
13015    #[test]
13016    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13017        assert!(round_is_clean(
13018            0,
13019            true,
13020            1,
13021            2,
13022            0,
13023            IncompleteReviewPolicy::Warn
13024        ));
13025    }
13026
13027    #[test]
13028    fn a_full_panel_with_an_open_finding_is_not_clean() {
13029        assert!(!round_is_clean(
13030            1,
13031            true,
13032            2,
13033            2,
13034            0,
13035            IncompleteReviewPolicy::Block
13036        ));
13037    }
13038
13039    #[test]
13040    fn a_full_panel_with_a_red_e2e_is_not_clean() {
13041        assert!(!round_is_clean(
13042            0,
13043            false,
13044            2,
13045            2,
13046            0,
13047            IncompleteReviewPolicy::Block
13048        ));
13049    }
13050
13051    // The stall this task closes: under the default `block` policy, a seat
13052    // missing only because it was rate limited must not force a wait for a
13053    // session limit that will not lift by the next round. `round_is_clean`
13054    // is where that quorum carve-out lives; the review loop around it never
13055    // changes what a reviewer's vote or a finding's severity means.
13056
13057    #[test]
13058    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13059        // 1 of 2 answered, and the one missing was quota'd — the exact
13060        // "review-2 rate limited (quota)" shape from the field report.
13061        assert!(round_is_clean(
13062            0,
13063            true,
13064            1,
13065            2,
13066            1,
13067            IncompleteReviewPolicy::Block
13068        ));
13069    }
13070
13071    #[test]
13072    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13073        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
13074        // not a quota loss (`quota_missing` stays 0) — worth another try.
13075        assert!(!round_is_clean(
13076            0,
13077            true,
13078            1,
13079            2,
13080            0,
13081            IncompleteReviewPolicy::Block
13082        ));
13083    }
13084
13085    #[test]
13086    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
13087        assert!(!round_is_clean(
13088            1,
13089            true,
13090            1,
13091            2,
13092            1,
13093            IncompleteReviewPolicy::Block
13094        ));
13095        assert!(!round_is_clean(
13096            0,
13097            false,
13098            1,
13099            2,
13100            1,
13101            IncompleteReviewPolicy::Block
13102        ));
13103    }
13104
13105    #[test]
13106    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
13107        // Every seat quota'd, nobody answered: there is no panel to decide
13108        // on, so this must fall through to the existing block-and-retry
13109        // fallback rather than call an unreviewed patch clean.
13110        assert!(!round_is_clean(
13111            0,
13112            true,
13113            0,
13114            2,
13115            2,
13116            IncompleteReviewPolicy::Block
13117        ));
13118    }
13119
13120    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
13121        CommandOutcome {
13122            command: "test".to_owned(),
13123            code,
13124            output_tail: String::new(),
13125            duration_ms: 0,
13126            resource_blocked,
13127        }
13128    }
13129
13130    #[test]
13131    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
13132        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
13133        assert!(
13134            !verify_inconclusive(&[outcome(Some(1), false)]),
13135            "an ordinary failure is still evidence about the patch"
13136        );
13137        assert!(verify_inconclusive(&[outcome(None, true)]));
13138        assert!(
13139            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
13140            "one inconclusive outcome taints the whole batch"
13141        );
13142        assert!(!verify_inconclusive(&[]));
13143    }
13144
13145    #[tokio::test]
13146    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
13147        // Alive for the first two checks, then dead - confirms the loop
13148        // actually re-polls rather than deciding once and sleeping out the
13149        // ceiling regardless.
13150        let calls = std::sync::atomic::AtomicUsize::new(0);
13151        let started = Instant::now();
13152        wait_for_pids_with(
13153            &[123],
13154            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
13155            Duration::from_millis(5),
13156            Duration::from_secs(5),
13157        )
13158        .await;
13159        assert!(
13160            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
13161            "must keep checking rather than deciding on the first answer"
13162        );
13163        assert!(
13164            started.elapsed() < Duration::from_secs(1),
13165            "must return the moment it is confirmed dead, not wait out the ceiling"
13166        );
13167    }
13168
13169    #[tokio::test]
13170    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
13171        let started = Instant::now();
13172        wait_for_pids_with(
13173            &[123],
13174            |_| true, // never reports dead
13175            Duration::from_millis(5),
13176            Duration::from_millis(30),
13177        )
13178        .await;
13179        let elapsed = started.elapsed();
13180        assert!(
13181            elapsed >= Duration::from_millis(30),
13182            "must not give up before its own ceiling: {elapsed:?}"
13183        );
13184        assert!(
13185            elapsed < Duration::from_secs(1),
13186            "must not wait past its own ceiling either: {elapsed:?}"
13187        );
13188    }
13189
13190    #[tokio::test]
13191    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
13192        let started = Instant::now();
13193        wait_for_pids_with(
13194            &[],
13195            |_| true,
13196            Duration::from_secs(5),
13197            Duration::from_secs(5),
13198        )
13199        .await;
13200        assert!(
13201            started.elapsed() < Duration::from_millis(200),
13202            "an empty pid list has nothing to confirm"
13203        );
13204    }
13205
13206    // `review_conclusion` is the exact decision the review hand-off task
13207    // fixed: a round budget spent (or a tree that stopped moving) must not
13208    // collapse into `Blocked` regardless of what verification actually
13209    // said. Deterministic and process-free for the same reason the
13210    // `round_is_clean` family above is.
13211    fn review_round(
13212        clean: bool,
13213        blocking: usize,
13214        answered: usize,
13215        expected: usize,
13216        progressed: bool,
13217        e2e_ok: bool,
13218    ) -> ReviewRound {
13219        ReviewRound {
13220            round: 1,
13221            head: "h".to_owned(),
13222            verified_head: None,
13223            verified_at: None,
13224            reviews: Vec::new(),
13225            e2e: vec![CommandOutcome {
13226                command: "test".to_owned(),
13227                code: Some(if e2e_ok { 0 } else { 1 }),
13228                output_tail: String::new(),
13229                duration_ms: 0,
13230                resource_blocked: false,
13231            }],
13232            verify_retried: false,
13233            e2e_deferred: false,
13234            e2e_defer_reason: None,
13235            fix: None,
13236            blocking,
13237            answered,
13238            expected,
13239            clean,
13240            progressed,
13241            vote_split: false,
13242            reconsideration: Vec::new(),
13243            verdict: None,
13244        }
13245    }
13246
13247    #[test]
13248    fn review_conclusion_is_none_when_nothing_has_run() {
13249        assert_eq!(review_conclusion(&[], 3), None);
13250    }
13251
13252    #[test]
13253    fn review_conclusion_is_none_while_rounds_remain() {
13254        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
13255        assert_eq!(review_conclusion(&rounds, 3), None);
13256    }
13257
13258    #[test]
13259    fn review_conclusion_is_gating_once_a_round_is_clean() {
13260        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
13261        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
13262    }
13263
13264    #[test]
13265    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
13266        let rounds = vec![
13267            review_round(false, 1, 2, 2, true, true),
13268            review_round(false, 1, 2, 2, true, true),
13269        ];
13270        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
13271    }
13272
13273    #[test]
13274    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
13275        let rounds = vec![
13276            review_round(false, 1, 2, 2, true, true),
13277            review_round(false, 1, 2, 2, true, false),
13278        ];
13279        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
13280    }
13281
13282    #[test]
13283    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
13284        // Magi never got a command to run against this round's own head — a
13285        // resource-blocked attempt, not a red one — so this must never
13286        // settle on `Blocked` the way a genuine e2e failure would. `None`
13287        // here is what tells `Runner::review_loop` to retry the check
13288        // itself rather than trust this cheap recomputation with a verdict
13289        // it cannot actually produce.
13290        let mut blocked = review_round(false, 1, 2, 2, true, false);
13291        blocked.e2e[0].resource_blocked = true;
13292        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
13293        assert_eq!(review_conclusion(&rounds, 2), None);
13294    }
13295
13296    #[test]
13297    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
13298        // Missing input, not a verified tree — never a hand-off candidate.
13299        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
13300        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
13301    }
13302
13303    #[test]
13304    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
13305        let rounds = vec![
13306            review_round(false, 1, 2, 2, false, true),
13307            review_round(false, 1, 2, 2, false, true),
13308        ];
13309        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
13310    }
13311
13312    fn secs(n: u64) -> Duration {
13313        Duration::from_secs(n)
13314    }
13315
13316    /// A throwaway repo with one commit on `main`, for tests that need `merge`
13317    /// to make real (and, if it runs at all, real*ly fail*) git calls.
13318    #[tokio::test]
13319    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13320        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
13321        if std::env::var_os(CHILD).is_none() {
13322            // Other tests reconcile every run in the process-global home,
13323            // briefly taking owner locks. This ancestry test expects an idle
13324            // predecessor, so give it its own process and pinned test home.
13325            let output = std::process::Command::new(std::env::current_exe().unwrap())
13326                .args([
13327                    "--exact",
13328                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
13329                    "--nocapture",
13330                ])
13331                .env(CHILD, "1")
13332                .env_remove("MAGI_WORKER_HOME")
13333                .env_remove("MAGI_WORKER_ID")
13334                .output()
13335                .unwrap();
13336            assert!(
13337                output.status.success()
13338                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
13339                "{}\n{}",
13340                String::from_utf8_lossy(&output.stdout),
13341                String::from_utf8_lossy(&output.stderr)
13342            );
13343            return;
13344        }
13345        ask_test_home();
13346        let tmp = tempfile::tempdir().unwrap();
13347        let repo = tmp.path().join("repo");
13348        std::fs::create_dir_all(&repo).unwrap();
13349        init_repo(&repo);
13350        let g = |args: &[&str]| {
13351            let out = std::process::Command::new("git")
13352                .args(args)
13353                .current_dir(&repo)
13354                .quiet()
13355                .output()
13356                .unwrap();
13357            assert!(
13358                out.status.success(),
13359                "{}",
13360                String::from_utf8_lossy(&out.stderr)
13361            );
13362        };
13363        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13364        g(&["checkout", "-b", "historical"]);
13365        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13366        g(&["add", "."]);
13367        g(&["commit", "-m", "old alternative"]);
13368        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13369        g(&["checkout", "-b", "magi/task-selected", &base]);
13370        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13371        g(&["add", "."]);
13372        g(&["commit", "-m", "measurement infrastructure"]);
13373        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13374        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13375        g(&["add", "."]);
13376        g(&["commit", "-m", "dependent implementation"]);
13377        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13378        g(&["checkout", "main"]);
13379        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13380        g(&["add", "."]);
13381        g(&["commit", "-m", "unrelated base change"]);
13382        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13383        let origin = tmp.path().join("origin.git");
13384        g(&[
13385            "clone",
13386            "--bare",
13387            repo.to_str().unwrap(),
13388            origin.to_str().unwrap(),
13389        ]);
13390        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13391        g(&["fetch", "origin"]);
13392        let selection = crate::pr_recovery::Selection {
13393            url: "https://example.test/project/pull/1".into(),
13394            branch: "magi/task-selected".into(),
13395            head: head.clone(),
13396            source_head: None,
13397            source_run: None,
13398            source_proof: None,
13399            disposition: Some(crate::pr_recovery::Disposition::Review),
13400            instruction: "preserve measured behavior".into(),
13401            approved_at: jiff::Timestamp::now(),
13402        };
13403        let mut template = runner_at(RunStatus::Reviewing);
13404        template.state.repo = repo.clone();
13405        template.state.base_commit = current.clone();
13406        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13407        template.state.origin = Some(crate::run::Origin::queue("selected"));
13408        template.state.pr_head = Some(selection.branch.clone());
13409        template.state.pr_recovery = Some(selection.clone());
13410        let mut runner =
13411            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13412                .await
13413                .unwrap();
13414        let wt = runner.state.winner().unwrap().worktree.clone();
13415        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13416        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13417        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13418        assert!(runner.state.seeds.is_empty());
13419        assert_eq!(runner.state.instruction, "preserve measured behavior");
13420        runner.sync_to_base().await.unwrap();
13421        assert_eq!(
13422            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13423            "prerequisite"
13424        );
13425        assert_eq!(
13426            std::fs::read_to_string(wt.join("implementation")).unwrap(),
13427            "measured implementation"
13428        );
13429        assert_eq!(
13430            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13431            "current main change"
13432        );
13433        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13434        assert_eq!(
13435            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13436            head
13437        );
13438
13439        // A failed selected-source setup stays inspectable, never resumable as a new competition.
13440        let mut partial = runner_at(RunStatus::Reviewing);
13441        partial.state.repo = repo.clone();
13442        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
13443        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
13444            run: Some("exit 1".into()),
13445            ..Default::default()
13446        }];
13447        let selection = runner.state.pr_recovery.clone().unwrap();
13448        partial.state.pr_recovery = Some(selection.clone());
13449        partial.state.pr_head = Some(selection.branch.clone());
13450        partial.state.origin = Some(crate::run::Origin::queue("selected"));
13451        let id = partial.state.id.clone();
13452        let error = match Runner::open_selected(
13453            &repo,
13454            partial.state,
13455            partial.roles,
13456            1,
13457            runner.state.base_commit.clone(),
13458            selection,
13459        )
13460        .await
13461        {
13462            Ok(_) => panic!("setup must fail"),
13463            Err(e) => e,
13464        };
13465        assert_eq!(
13466            error
13467                .downcast_ref::<crate::pr_recovery::StartupFailed>()
13468                .unwrap()
13469                .run,
13470            id
13471        );
13472        let failed = RunState::load(&id).unwrap();
13473        assert_eq!(failed.status, RunStatus::Blocked);
13474        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
13475        assert!(Runner::resume(&id).is_err());
13476        runner.state.status = RunStatus::Ready;
13477        runner.state.driver_exited = true;
13478        runner.state.save().unwrap();
13479        let mut retry = runner_at(RunStatus::Reviewing);
13480        retry.state.repo = repo.clone();
13481        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
13482        let mut reselected = runner.state.pr_recovery.clone().unwrap();
13483        reselected.approved_at = jiff::Timestamp::now();
13484        retry.state.origin = Some(crate::run::Origin::queue("selected"));
13485        retry.state.pr_head = Some(reselected.branch.clone());
13486        retry.state.pr_recovery = Some(reselected.clone());
13487        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
13488            .await
13489            .unwrap();
13490        let restarted = Runner::open_selected(
13491            &repo,
13492            retry.state,
13493            retry.roles,
13494            1,
13495            runner.state.base_commit.clone(),
13496            reselected,
13497        )
13498        .await
13499        .unwrap();
13500        assert_eq!(
13501            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
13502                .await
13503                .unwrap(),
13504            head
13505        );
13506        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13507        assert_eq!(
13508            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13509            head
13510        );
13511    }
13512
13513    #[tokio::test]
13514    #[cfg(unix)]
13515    async fn fallback_and_interrupted_restoration_require_coordination() {
13516        use std::os::fd::AsRawFd as _;
13517        ask_test_home();
13518        let tmp = tempfile::tempdir().unwrap();
13519        let mut runner = registered_runner(tmp.path()).await;
13520        runner.state.config.graph.timeout_resource_wait = 1;
13521        let winner = runner.state.candidates[0].clone();
13522        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13523        let leftover = winner.worktree.join("leftover");
13524        std::fs::write(&leftover, "untrusted edits").unwrap();
13525        let next = tmp.path().join("next-fixer");
13526        let mut failed = spec("first");
13527        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
13528        let mut successor = spec("second");
13529        successor.command = vec![
13530            "sh".into(),
13531            "-c".into(),
13532            format!("touch '{}'; echo done", next.display()),
13533        ];
13534        let attempts = vec![
13535            (failed.clone(), "first".into()),
13536            (successor.clone(), "second".into()),
13537        ];
13538        let lock_path = runner.state.repo.join(".git/updater.lock");
13539        let external = std::fs::File::create(&lock_path).unwrap();
13540        assert_eq!(
13541            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13542            0
13543        );
13544        let artifacts = tmp.path().join("artifacts");
13545        let build = |spec, seat| SeatJob {
13546            spec,
13547            seat,
13548            cwd: winner.worktree.clone(),
13549            prompt: "fixture".into(),
13550            timeout: Duration::from_secs(10),
13551            allow_write: true,
13552            sessions: false,
13553            artifacts: artifacts.clone(),
13554            stem: "fix".into(),
13555            handover: None,
13556        };
13557        let (_, _, out) = runner
13558            .ask_fixer_among(
13559                attempts.clone(),
13560                "graft",
13561                None,
13562                Some((&winner.worktree, &head)),
13563                build,
13564            )
13565            .await;
13566        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13567        assert!(leftover.exists(), "contention must prevent clean");
13568        assert!(!next.exists());
13569        assert!(runner.state.handovers.is_empty());
13570        // Early interrupted-repair rollback uses the same protected route.
13571        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13572            from_run: "earlier".into(),
13573            generation: 1,
13574            mode: crate::recovery::Mode::VerifyRepair,
13575            reasons: Vec::new(),
13576            reviewed_head: None,
13577            rounds: Vec::new(),
13578            open_findings: Vec::new(),
13579            evidence: None,
13580            criteria_inherited: true,
13581            repair: crate::recovery::Repair::Started {
13582                before: head.clone(),
13583            },
13584        });
13585        assert!(
13586            runner
13587                .recovery_repair(&winner)
13588                .await
13589                .unwrap_err()
13590                .is::<crate::repo_lock::ResourceBlocked>()
13591        );
13592        assert!(matches!(
13593            runner.state.recovery.as_ref().unwrap().repair,
13594            crate::recovery::Repair::Started { .. }
13595        ));
13596        assert!(leftover.exists());
13597        drop(external);
13598        runner.recovery_repair(&winner).await.unwrap();
13599        assert!(!leftover.exists());
13600        assert!(matches!(
13601            runner.state.recovery.as_ref().unwrap().repair,
13602            crate::recovery::Repair::Done { .. }
13603        ));
13604        // A hard lock-open failure also leaves HEAD and untracked work intact.
13605        std::fs::remove_file(&lock_path).unwrap();
13606        std::fs::create_dir(&lock_path).unwrap();
13607        std::fs::write(&leftover, "untrusted edits").unwrap();
13608        let (_, _, out) = runner
13609            .ask_fixer_among(
13610                attempts,
13611                "graft",
13612                None,
13613                Some((&winner.worktree, &head)),
13614                build,
13615            )
13616            .await;
13617        assert!(matches!(out, AgentOutcome::Failed(_)));
13618        assert!(leftover.exists());
13619        assert!(!next.exists());
13620        assert_eq!(
13621            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13622            head
13623        );
13624    }
13625
13626    #[cfg(unix)]
13627    async fn registered_runner(root: &Path) -> Runner {
13628        ask_test_home();
13629        let repo = root.join("repo");
13630        std::fs::create_dir_all(&repo).unwrap();
13631        init_repo(&repo);
13632        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13633        let mut runner = runner_at(RunStatus::Prep);
13634        runner.state.repo = repo.clone();
13635        runner.state.config.coordination = crate::config::test_coordination();
13636        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13637        runner.state.config.graph.worktree_root = Some(root.join("wt"));
13638        runner.state.config.graph.advise = false;
13639        runner.roles.implementers = vec![spec("implement")];
13640        runner.roles.judges = vec![spec("judge")];
13641        runner.prep().await.unwrap();
13642        runner
13643    }
13644
13645    #[tokio::test]
13646    #[cfg(unix)]
13647    async fn parked_implementer_persists_completed_work() {
13648        for registered in [false, true] {
13649            let tmp = tempfile::tempdir().unwrap();
13650            let mut runner = registered_runner(tmp.path()).await;
13651            if !registered {
13652                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13653            }
13654            let marker = tmp.path().join("started");
13655            let release = tmp.path().join("release");
13656            let mut implementer = spec("implement");
13657            implementer.command = vec![
13658                "sh".into(),
13659                "-c".into(),
13660                format!(
13661                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13662                    marker.display(),
13663                    release.display()
13664                ),
13665            ];
13666            runner.roles.implementers = vec![implementer.clone()];
13667            runner.state.config.agents = vec![implementer];
13668            let pause = runner.pause.clone();
13669            let (result, ()) = tokio::join!(runner.implement(), async {
13670                tokio::time::timeout(Duration::from_secs(10), async {
13671                    while !marker.exists() {
13672                        tokio::time::sleep(Duration::from_millis(10)).await;
13673                    }
13674                })
13675                .await
13676                .unwrap();
13677                pause.park();
13678                std::fs::write(&release, "go").unwrap();
13679            });
13680            result.unwrap();
13681            let candidate = &runner.state.candidates[0];
13682            assert!(candidate.commits > 0, "{candidate:?}");
13683            assert!(candidate.summary.contains("saved-result"));
13684            assert_eq!(candidate.files, 1);
13685            let before = candidate.commits;
13686            runner.implement().await.unwrap();
13687            assert_eq!(runner.state.candidates[0].commits, before);
13688        }
13689    }
13690
13691    #[tokio::test]
13692    #[cfg(unix)]
13693    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13694        let tmp = tempfile::tempdir().unwrap();
13695        let mut runner = registered_runner(tmp.path()).await;
13696        let winner = runner.state.candidates[0].clone();
13697        let cache = tmp.path().join("cache");
13698        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13699        runner.state.config.graph.timeout_resource_wait = 0;
13700        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13701        let crate::cache::AcquireOutcome::Acquired(held) =
13702            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13703        else {
13704            panic!("cache must be free")
13705        };
13706        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13707            from_run: "earlier".into(),
13708            generation: 1,
13709            mode: crate::recovery::Mode::VerifyRepair,
13710            reasons: Vec::new(),
13711            reviewed_head: None,
13712            rounds: Vec::new(),
13713            open_findings: Vec::new(),
13714            evidence: None,
13715            criteria_inherited: true,
13716            repair: crate::recovery::Repair::NotStarted,
13717        });
13718        runner.recovery_repair(&winner).await.unwrap();
13719        assert!(matches!(
13720            runner.state.recovery.as_ref().unwrap().repair,
13721            crate::recovery::Repair::WaitingForRestore { .. }
13722        ));
13723        drop(held);
13724        runner.state.resource_block = None;
13725        runner.state.config.graph.timeout_resource_wait = 1;
13726        runner.recovery_repair(&winner).await.unwrap();
13727        assert!(matches!(
13728            runner.state.recovery.as_ref().unwrap().repair,
13729            crate::recovery::Repair::Done { .. }
13730        ));
13731        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13732    }
13733
13734    #[tokio::test]
13735    #[cfg(unix)]
13736    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13737        use std::os::fd::AsRawFd as _;
13738        for graft in [false, true] {
13739            let tmp = tempfile::tempdir().unwrap();
13740            let mut runner = registered_runner(tmp.path()).await;
13741            runner.state.config.graph.timeout_resource_wait = 1;
13742            let winner = runner.state.candidates[0].clone();
13743            let marker = tmp.path().join("fixer-started");
13744            let mut fixer = spec("fixer");
13745            fixer.command = vec![
13746                "sh".into(),
13747                "-c".into(),
13748                format!("echo started >> '{}'; echo done", marker.display()),
13749            ];
13750            runner.roles.fixer = Some(vec![fixer]);
13751            if graft {
13752                let mut loser = winner.clone();
13753                loser.label = 'B';
13754                loser.index = 1;
13755                runner.state.candidates.push(loser);
13756                runner.state.config.graph.graft_runner_up = true;
13757                runner.state.tally = Some(Tally {
13758                    first_choice: BTreeMap::new(),
13759                    borda: BTreeMap::new(),
13760                    winner: winner.label,
13761                    rankings: 1,
13762                    unanimous_initial: true,
13763                    deliberated: false,
13764                    changed_votes: 0,
13765                    unanimous_final: true,
13766                    tie_break: None,
13767                    judges: 1,
13768                    present: 1,
13769                    quorum: 1,
13770                    met_quorum: true,
13771                    uncontested: None,
13772                });
13773                runner.state.judgements.push(crate::run::Judgement {
13774                    judge: 1,
13775                    seat: "judge-1".into(),
13776                    agent: "judge".into(),
13777                    ranking: vec![winner.label, 'B'],
13778                    reasons: BTreeMap::new(),
13779                    confidence: None,
13780                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13781                    order: vec![0, 1],
13782                    failed: None,
13783                    duration_ms: 0,
13784                });
13785            } else {
13786                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13787                    from_run: "earlier".into(),
13788                    generation: 1,
13789                    mode: crate::recovery::Mode::VerifyRepair,
13790                    reasons: Vec::new(),
13791                    reviewed_head: None,
13792                    rounds: Vec::new(),
13793                    open_findings: Vec::new(),
13794                    evidence: None,
13795                    criteria_inherited: true,
13796                    repair: crate::recovery::Repair::NotStarted,
13797                });
13798            }
13799            let reader = crate::repo_lock::SeatLease::for_checkout(
13800                &winner.worktree,
13801                false,
13802                crate::config::test_coordination().as_ref(),
13803            )
13804            .await
13805            .unwrap();
13806            let pause = runner.pause.clone();
13807            let (result, ()) = tokio::join!(
13808                async {
13809                    if graft {
13810                        runner.graft().await
13811                    } else {
13812                        runner.recovery_repair(&winner).await
13813                    }
13814                },
13815                async {
13816                    tokio::time::sleep(Duration::from_millis(100)).await;
13817                    pause.park();
13818                }
13819            );
13820            result.unwrap();
13821            assert!(!marker.exists(), "a blocked seat must not spawn");
13822            // Exercise the persisted state, as a binary replacement/resume does.
13823            runner.state = RunState::load(&runner.state.id).unwrap();
13824            if graft {
13825                assert_eq!(
13826                    runner.state.graft.as_ref().unwrap().outcome,
13827                    GraftOutcome::WaitingForRestore
13828                );
13829            } else {
13830                assert!(matches!(
13831                    runner.state.recovery.as_ref().unwrap().repair,
13832                    crate::recovery::Repair::WaitingForRestore { .. }
13833                ));
13834            }
13835            drop(reader);
13836            runner.pause = Pause::new();
13837            runner.state.resource_block = None;
13838            // Deferred restoration must still respect an external lock owner.
13839            let external =
13840                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13841            assert_eq!(
13842                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13843                0
13844            );
13845            let leftover = winner.worktree.join("leftover");
13846            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13847            let result = if graft {
13848                runner.graft().await
13849            } else {
13850                runner.recovery_repair(&winner).await
13851            };
13852            assert!(
13853                result
13854                    .unwrap_err()
13855                    .is::<crate::repo_lock::ResourceBlocked>()
13856            );
13857            assert!(leftover.exists());
13858            assert!(!marker.exists());
13859            drop(external);
13860            runner.state.resource_block = None;
13861            let result = if graft {
13862                runner.graft().await
13863            } else {
13864                runner.recovery_repair(&winner).await
13865            };
13866            result.unwrap();
13867            assert!(!leftover.exists(), "restore must precede retry");
13868            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13869            if graft {
13870                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13871            } else {
13872                assert!(matches!(
13873                    runner.state.recovery.as_ref().unwrap().repair,
13874                    crate::recovery::Repair::Done { .. }
13875                ));
13876            }
13877        }
13878    }
13879
13880    #[tokio::test]
13881    #[cfg(unix)]
13882    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13883        use std::os::fd::AsRawFd as _;
13884        let tmp = tempfile::tempdir().unwrap();
13885        let mut runner = registered_runner(tmp.path()).await;
13886        runner.state.config.graph.timeout_resource_wait = 1;
13887        let winner = runner.state.candidates[0].clone();
13888        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13889        runner.state.tally = Some(Tally {
13890            first_choice: BTreeMap::new(),
13891            borda: BTreeMap::new(),
13892            winner: winner.label,
13893            rankings: 0,
13894            unanimous_initial: false,
13895            deliberated: false,
13896            changed_votes: 0,
13897            unanimous_final: false,
13898            tie_break: None,
13899            judges: 0,
13900            present: 0,
13901            quorum: 0,
13902            met_quorum: true,
13903            uncontested: Some("fixture".into()),
13904        });
13905        runner.state.graft = Some(GraftRecord {
13906            pre: pre.clone(),
13907            post: None,
13908            lines: None,
13909            limit: None,
13910            agent: None,
13911            outcome: GraftOutcome::Running,
13912            requested: Vec::new(),
13913            adoption: None,
13914            adoption_error: None,
13915        });
13916        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13917        git::commit_all(&winner.worktree, "unfinished graft")
13918            .await
13919            .unwrap();
13920        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13921        let leftover = winner.worktree.join("leftover");
13922        std::fs::write(&leftover, "untrusted edits").unwrap();
13923        let path = runner.state.repo.join(".git/updater.lock");
13924        let external = std::fs::File::create(&path).unwrap();
13925        assert_eq!(
13926            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13927            0
13928        );
13929        assert!(
13930            runner
13931                .graft()
13932                .await
13933                .unwrap_err()
13934                .is::<crate::repo_lock::ResourceBlocked>()
13935        );
13936        assert!(matches!(
13937            runner.state.graft.as_ref().unwrap().outcome,
13938            GraftOutcome::Running
13939        ));
13940        assert_eq!(
13941            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13942            unfinished
13943        );
13944        assert!(leftover.exists());
13945        drop(external);
13946        std::fs::remove_file(&path).unwrap();
13947        std::fs::create_dir(&path).unwrap();
13948        assert!(runner.graft().await.is_err());
13949        assert!(matches!(
13950            runner.state.graft.as_ref().unwrap().outcome,
13951            GraftOutcome::Running
13952        ));
13953        assert_eq!(
13954            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13955            unfinished
13956        );
13957        assert!(leftover.exists());
13958        std::fs::remove_dir(&path).unwrap();
13959        runner.graft().await.unwrap();
13960        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13961        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13962        assert!(!leftover.exists());
13963    }
13964
13965    #[tokio::test]
13966    #[cfg(unix)]
13967    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13968        use std::os::fd::AsRawFd as _;
13969        let tmp = tempfile::tempdir().unwrap();
13970        let mut runner = registered_runner(tmp.path()).await;
13971        runner.state.prep_complete = Some(false);
13972        runner.state.config.graph.timeout_resource_wait = 1;
13973        runner.state.save().unwrap();
13974        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13975        assert_eq!(
13976            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13977            0
13978        );
13979        runner.execute().await.unwrap();
13980        let disk = RunState::load(&runner.state.id).unwrap();
13981        assert!(disk.driver_exited);
13982        assert_eq!(disk.driver_pid, None);
13983        assert_eq!(runner.state.status, RunStatus::Blocked);
13984        assert_eq!(
13985            runner.state.resource_block.as_ref().unwrap().kind,
13986            "repository coordination"
13987        );
13988        assert!(runner.state.prep_failure.is_none());
13989        assert!(runner.state.seats.is_empty());
13990        runner.pause.park();
13991        // Include headroom for Git subprocesses under parallel test load;
13992        // the repository contention deadline itself remains one second.
13993        tokio::time::timeout(Duration::from_secs(10), runner.execute())
13994            .await
13995            .unwrap()
13996            .unwrap();
13997        assert!(runner.state.parked);
13998        assert!(runner.state.prep_failure.is_none());
13999        let disk = RunState::load(&runner.state.id).unwrap();
14000        assert!(disk.driver_exited);
14001        assert_eq!(disk.driver_pid, None);
14002    }
14003
14004    #[tokio::test]
14005    #[cfg(unix)]
14006    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
14007        use std::os::fd::AsRawFd as _;
14008        let tmp = tempfile::tempdir().unwrap();
14009        let mut runner = registered_runner(tmp.path()).await;
14010        runner.state.prep_complete = Some(false);
14011        runner.state.config.graph.timeout_resource_wait = 30;
14012        runner.state.save().unwrap();
14013        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14014        assert_eq!(
14015            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14016            0
14017        );
14018        assert!(
14019            tokio::time::timeout(Duration::from_millis(100), runner.execute())
14020                .await
14021                .is_err()
14022        );
14023        let disk = RunState::load(&runner.state.id).unwrap();
14024        assert!(disk.driver_exited);
14025        assert_eq!(disk.driver_pid, None);
14026        assert!(disk.status.resumable());
14027        assert!(runner.state.driver_exited);
14028        assert_eq!(runner.state.driver_pid, None);
14029        runner.state.save().unwrap();
14030        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
14031    }
14032
14033    #[tokio::test]
14034    #[cfg(unix)]
14035    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
14036        let tmp = tempfile::tempdir().unwrap();
14037        let mut runner = registered_runner(tmp.path()).await;
14038        let winner = runner.state.candidates[0].clone();
14039        let cache = tmp.path().join("cache");
14040        let started = tmp.path().join("pre-gate-started");
14041        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
14042        runner.state.config.verify.gate =
14043            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
14044        let home = crate::run::home();
14045        let owner =
14046            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
14047        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
14048            crate::cache::AcquireOutcome::Acquired(g) => g,
14049            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
14050        };
14051        let mut gate = Box::pin(runner.run_gate(&winner));
14052        assert!(
14053            tokio::time::timeout(Duration::from_millis(150), &mut gate)
14054                .await
14055                .is_err()
14056        );
14057        assert!(!started.exists(), "pre-gate must wait for the cache too");
14058        // Simulate the wave that already owns the cache entering invoke.
14059        // It must obtain the editor reservation and finish without waiting
14060        // for the gate that is queued behind its cache.
14061        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
14062        let editor = crate::repo_lock::SeatLease::checkout_with(
14063            &winner.worktree,
14064            true,
14065            crate::config::test_coordination().as_ref(),
14066            &wait,
14067        )
14068        .await
14069        .unwrap();
14070        drop(editor);
14071        guard.release();
14072        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14073            .await
14074            .unwrap()
14075            .unwrap();
14076        drop(gate);
14077        assert!(outcomes.iter().all(CommandOutcome::ok));
14078        assert!(started.exists());
14079    }
14080
14081    #[tokio::test]
14082    #[cfg(unix)]
14083    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
14084        use std::os::fd::AsRawFd as _;
14085
14086        ask_test_home();
14087        let tmp = tempfile::tempdir().unwrap();
14088        let repo = tmp.path().join("repo");
14089        std::fs::create_dir_all(&repo).unwrap();
14090        init_repo(&repo);
14091        let common = repo.join(".git");
14092        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14093        let mut runner = runner_at(RunStatus::Prep);
14094        runner.state.repo = repo.clone();
14095        runner.state.config.coordination = crate::config::test_coordination();
14096        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14097        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14098        runner.state.config.graph.advise = false;
14099        runner.roles.implementers = vec![spec("implement")];
14100        runner.roles.judges = vec![spec("judge")];
14101        runner.prep().await.unwrap();
14102        let winner = runner.state.candidates[0].clone();
14103        // Emulate edits already left by a completed seat. The fixture CLI
14104        // only reports completion; it performs no repository operations.
14105        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
14106        let completed = tmp.path().join("seat-completed");
14107        runner.roles.implementers[0].command = vec![
14108            "sh".into(),
14109            "-c".into(),
14110            format!("touch '{}'; echo completed", completed.display()),
14111        ];
14112        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
14113        assert_eq!(
14114            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14115            0
14116        );
14117        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14118        let mut implement = Box::pin(runner.implement());
14119        tokio::select! {
14120            result = &mut implement => panic!("rescue must wait: {result:?}"),
14121            result = tokio::time::timeout(Duration::from_secs(10), async {
14122                while !completed.exists() {
14123                    tokio::time::sleep(Duration::from_millis(20)).await;
14124                }
14125            }) => result.unwrap(),
14126        }
14127        assert!(
14128            tokio::time::timeout(Duration::from_millis(200), &mut implement)
14129                .await
14130                .is_err()
14131        );
14132        assert_eq!(
14133            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14134            before
14135        );
14136        assert_eq!(
14137            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14138            0
14139        );
14140        tokio::time::timeout(Duration::from_secs(10), &mut implement)
14141            .await
14142            .unwrap()
14143            .unwrap();
14144        drop(implement);
14145        assert_ne!(
14146            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14147            before
14148        );
14149        assert!(git::is_clean(&winner.worktree).await.unwrap());
14150
14151        let started = tmp.path().join("pre-gate-started");
14152        let proceed = tmp.path().join("pre-gate-proceed");
14153        let checked = tmp.path().join("gate-ran");
14154        runner.state.config.verify.pre_gate = vec![format!(
14155            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14156            started.display(),
14157            proceed.display()
14158        )];
14159        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
14160        assert_eq!(
14161            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14162            0
14163        );
14164        let mut gate = Box::pin(runner.run_gate(&winner));
14165        assert!(
14166            tokio::time::timeout(Duration::from_millis(200), &mut gate)
14167                .await
14168                .is_err()
14169        );
14170        assert!(!started.exists());
14171        assert!(!checked.exists());
14172        assert!(git::is_clean(&winner.worktree).await.unwrap());
14173        assert_eq!(
14174            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14175            0
14176        );
14177        tokio::select! {
14178            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
14179            result = tokio::time::timeout(Duration::from_secs(10), async {
14180                while !started.exists() {
14181                    tokio::time::sleep(Duration::from_millis(20)).await;
14182                }
14183            }) => result.unwrap(),
14184        }
14185        assert_eq!(
14186            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14187            -1,
14188            "mechanical edits must hold the real lock"
14189        );
14190        std::fs::write(proceed, "ready").unwrap();
14191        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14192            .await
14193            .unwrap()
14194            .unwrap();
14195        drop(gate);
14196        assert!(outcomes.iter().all(CommandOutcome::ok));
14197        assert!(checked.exists());
14198        assert!(runner.state.pre_gate_commit.is_some());
14199        assert!(git::is_clean(&winner.worktree).await.unwrap());
14200        assert_eq!(
14201            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14202            0,
14203            "gate completion must release coordination"
14204        );
14205        drop(outsider);
14206        std::fs::remove_file(common.join("updater.lock")).unwrap();
14207        std::fs::create_dir(common.join("updater.lock")).unwrap();
14208        std::fs::remove_file(&started).unwrap();
14209        std::fs::remove_file(&checked).unwrap();
14210        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14211        let error = runner.run_gate(&winner).await.unwrap_err();
14212        assert!(
14213            error
14214                .to_string()
14215                .contains("acquire mandatory repository lock")
14216        );
14217        assert!(
14218            !started.exists(),
14219            "failed acquisition must prevent pre-gate"
14220        );
14221        assert!(!checked.exists(), "failed acquisition must prevent gate");
14222        assert!(git::is_clean(&winner.worktree).await.unwrap());
14223        assert_eq!(
14224            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14225            head
14226        );
14227    }
14228
14229    #[tokio::test]
14230    #[cfg(unix)]
14231    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
14232        use std::os::fd::AsRawFd as _;
14233        use std::time::Duration;
14234
14235        ask_test_home();
14236        let tmp = tempfile::tempdir().unwrap();
14237        let repo = tmp.path().join("repo");
14238        std::fs::create_dir_all(&repo).unwrap();
14239        init_repo(&repo);
14240        let common = repo.join(".git");
14241        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14242        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
14243        assert_eq!(
14244            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14245            0
14246        );
14247        let mut runner = runner_at(RunStatus::Prep);
14248        runner.state.repo = repo.clone();
14249        runner.state.config.coordination = crate::config::test_coordination();
14250        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14251        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14252        runner.state.config.graph.advise = true;
14253        runner.state.config.graph.advisors = 1;
14254        runner.roles.implementers = vec![spec("implement")];
14255        runner.roles.judges = vec![spec("judge")];
14256        let root = runner.state.worktree_root();
14257        let started = tmp.path().join("setup-started");
14258        let proceed = tmp.path().join("setup-proceed");
14259        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14260            run: Some(format!(
14261                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14262                started.display(),
14263                proceed.display()
14264            )),
14265            ..Default::default()
14266        }];
14267        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
14268        let config = git::git(&repo, &["config", "--local", "--list"])
14269            .await
14270            .unwrap();
14271        let mut prep = Box::pin(runner.prep());
14272        assert!(
14273            tokio::time::timeout(Duration::from_millis(200), &mut prep)
14274                .await
14275                .is_err()
14276        );
14277        assert!(
14278            !root.exists(),
14279            "contention must prevent even checkout creation"
14280        );
14281        assert!(!started.exists(), "contention must prevent setup");
14282        assert_eq!(
14283            git::git(&repo, &["branch", "--list"]).await.unwrap(),
14284            branches
14285        );
14286        assert_eq!(
14287            git::git(&repo, &["config", "--local", "--list"])
14288                .await
14289                .unwrap(),
14290            config
14291        );
14292        drop(lock);
14293        tokio::select! {
14294            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
14295            result = tokio::time::timeout(Duration::from_secs(10), async {
14296                while !started.exists() {
14297                    tokio::time::sleep(Duration::from_millis(20)).await;
14298                }
14299            }) => result.unwrap(),
14300        }
14301        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
14302        assert_eq!(
14303            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14304            -1
14305        );
14306        assert!(
14307            tokio::time::timeout(
14308                Duration::from_millis(100),
14309                crate::repo_lock::SeatLease::for_checkout(
14310                    &repo,
14311                    true,
14312                    crate::config::test_coordination().as_ref()
14313                )
14314            )
14315            .await
14316            .is_err(),
14317            "preparation must also exclude local editors"
14318        );
14319        std::fs::write(proceed, "ready").unwrap();
14320        tokio::time::timeout(Duration::from_secs(10), &mut prep)
14321            .await
14322            .unwrap()
14323            .unwrap();
14324        drop(prep);
14325        assert_eq!(runner.state.prep_complete, Some(true));
14326        assert!(root.join("judge-1").exists());
14327        assert!(root.join("advisor-1").exists());
14328        assert_eq!(
14329            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14330            0
14331        );
14332    }
14333
14334    #[tokio::test]
14335    #[cfg(unix)]
14336    async fn preparation_lock_failure_prevents_repository_changes() {
14337        ask_test_home();
14338        let tmp = tempfile::tempdir().unwrap();
14339        let repo = tmp.path().join("repo");
14340        std::fs::create_dir_all(&repo).unwrap();
14341        init_repo(&repo);
14342        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14343        // A directory at the required lock path makes opening the real lock
14344        // fail, rather than merely exercising an invalid registration.
14345        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
14346        let mut runner = runner_at(RunStatus::Prep);
14347        runner.state.repo = repo.clone();
14348        runner.state.config.coordination = crate::config::test_coordination();
14349        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14350        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14351        runner.roles.implementers = vec![spec("implement")];
14352        let error = runner.prep().await.unwrap_err();
14353        assert!(
14354            error
14355                .to_string()
14356                .contains("acquire mandatory repository lock")
14357        );
14358        assert!(!runner.state.worktree_root().exists());
14359        assert!(runner.state.candidates.is_empty());
14360        assert!(!repo.join(".git/config.worktree").exists());
14361        assert_eq!(
14362            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
14363            "* main"
14364        );
14365    }
14366
14367    #[tokio::test]
14368    async fn preparation_records_creation_intent_before_git_can_fail() {
14369        ask_test_home();
14370        let tmp = tempfile::tempdir().unwrap();
14371        let repo = tmp.path().join("repo");
14372        std::fs::create_dir_all(&repo).unwrap();
14373        init_repo(&repo);
14374        let mut runner = runner_at(RunStatus::Prep);
14375        runner.state.repo = repo.clone();
14376        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14377        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14378        runner.state.config.blind.commit_msg_hook = false;
14379        runner.roles.implementers = vec![spec("one")];
14380        let label = blind::assign_labels(1, runner.state.seed)[0];
14381        let branch = runner.state.branch_for(label);
14382        // A conflicting branch makes Git refuse creation before a candidate
14383        // can be registered. The durable intent must still identify it.
14384        git::git(&repo, &["branch", &branch]).await.unwrap();
14385        assert!(runner.prep().await.is_err());
14386        let saved = RunState::load(&runner.state.id).unwrap();
14387        assert!(saved.candidates.is_empty());
14388        let event = saved
14389            .events
14390            .iter()
14391            .find(|e| e.node == "prep-intent")
14392            .unwrap();
14393        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
14394        assert_eq!(intent["branch"], branch);
14395        assert_eq!(intent["start"], runner.state.base_commit);
14396        assert_eq!(
14397            intent["worktree"],
14398            runner
14399                .state
14400                .worktree_root()
14401                .join(format!("cand-{label}"))
14402                .to_string_lossy()
14403                .as_ref()
14404        );
14405        assert_eq!(intent["picks"], serde_json::json!([]));
14406    }
14407
14408    #[tokio::test]
14409    async fn invalid_pinned_sources_create_no_partial_worktrees() {
14410        ask_test_home();
14411        let tmp = tempfile::tempdir().unwrap();
14412        let repo = tmp.path().join("repo");
14413        std::fs::create_dir_all(&repo).unwrap();
14414        init_repo(&repo);
14415        let mut runner = runner_at(RunStatus::Prep);
14416        runner.state.repo = repo.clone();
14417        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14418        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14419        runner.state.sources.pick_commits.push(refs::Input {
14420            token: "missing".into(),
14421            sha: "0".repeat(40),
14422            provenance: refs::Provenance::ExplicitCli,
14423        });
14424        assert!(runner.prep().await.is_err());
14425        assert!(runner.state.candidates.is_empty());
14426        assert!(!runner.state.worktree_root().exists());
14427    }
14428
14429    #[tokio::test]
14430    async fn prose_and_legacy_inference_never_import_into_candidates() {
14431        ask_test_home();
14432        let tmp = tempfile::tempdir().unwrap();
14433        let repo = tmp.path().join("repo");
14434        std::fs::create_dir_all(&repo).unwrap();
14435        init_repo(&repo);
14436        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14437        git::git(&repo, &["checkout", "-b", "magi/history/A"])
14438            .await
14439            .unwrap();
14440        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
14441        git::git(&repo, &["add", "."]).await.unwrap();
14442        git::git(&repo, &["commit", "-m", "historical implementation"])
14443            .await
14444            .unwrap();
14445        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
14446        git::git(&repo, &["checkout", "main"]).await.unwrap();
14447        for instruction in [
14448            format!("Example: magi/history/A at {source}"),
14449            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
14450            format!("Never import magi/history/A or {source}"),
14451            format!("Conversation answer: I mentioned {source} only as background"),
14452        ] {
14453            let mut runner = runner_at(RunStatus::Prep);
14454            runner.state.repo = repo.clone();
14455            runner.state.base_commit = base.clone();
14456            runner.state.instruction = instruction.clone();
14457            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
14458            runner.state.config.blind.commit_msg_hook = false;
14459            runner.state.seeds = vec![refs::Seed {
14460                token: "magi/history/A".into(),
14461                sha: source.clone(),
14462                kind: refs::SeedKind::Unmerged,
14463                branch: true,
14464                detail: String::new(),
14465            }];
14466            runner.roles.implementers = vec![spec("one")];
14467            assert_eq!(seeded_instruction(&runner.state), instruction);
14468            // Model an old record with inferred seeds but no structured input field.
14469            let mut legacy = serde_json::to_value(&runner.state).unwrap();
14470            legacy.as_object_mut().unwrap().remove("sources");
14471            runner.state = serde_json::from_value(legacy).unwrap();
14472            assert_eq!(runner.state.sources, refs::Sources::default());
14473            runner.state.save().unwrap();
14474            runner.state = RunState::load(&runner.state.id).unwrap();
14475            runner.prep().await.unwrap();
14476            for candidate in &runner.state.candidates {
14477                assert_eq!(
14478                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
14479                    base
14480                );
14481                assert!(!candidate.worktree.join("unwanted").exists());
14482            }
14483        }
14484    }
14485
14486    #[tokio::test]
14487    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
14488        ask_test_home();
14489        let tmp = tempfile::tempdir().unwrap();
14490        let repo = tmp.path().join("repo");
14491        std::fs::create_dir_all(&repo).unwrap();
14492        init_repo(&repo);
14493        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14494        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
14495        std::fs::write(repo.join("referenced"), "required commit").unwrap();
14496        git::git(&repo, &["add", "."]).await.unwrap();
14497        git::git(&repo, &["commit", "-m", "required seed"])
14498            .await
14499            .unwrap();
14500        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
14501        git::git(&repo, &["checkout", "main"]).await.unwrap();
14502        let mut runner = runner_at(RunStatus::Prep);
14503        runner.state.repo = repo.clone();
14504        runner.state.base_commit = base.clone();
14505        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14506        runner.state.config.graph.advise = false;
14507        runner.state.config.blind.commit_msg_hook = false;
14508        runner.state.sources =
14509            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
14510                .await
14511                .unwrap();
14512        runner.roles.implementers = vec![spec("one"), spec("two")];
14513        runner.roles.judges = vec![spec("judge")];
14514        let label = blind::assign_labels(2, runner.state.seed)[0];
14515        let branch = runner.state.branch_for(label);
14516        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14517        git::worktree_add_branch(&repo, &worktree, &branch, &base)
14518            .await
14519            .unwrap();
14520        runner.state.candidates.push(Candidate {
14521            index: 0,
14522            label,
14523            agent: "one".into(),
14524            branch,
14525            worktree,
14526            summary: String::new(),
14527            stat: String::new(),
14528            files: 0,
14529            commits: 0,
14530            empty: false,
14531            failed: None,
14532            verified_noop: None,
14533            duration_ms: 0,
14534            folded: false,
14535        });
14536        runner.state.save().unwrap();
14537        // Reload the checkpoint immediately before its first cherry-pick.
14538        runner.state = RunState::load(&runner.state.id).unwrap();
14539        assert_eq!(runner.state.prep_complete, Some(false));
14540        runner.prep().await.unwrap();
14541        assert_eq!(runner.state.prep_complete, Some(true));
14542        assert_eq!(runner.state.candidates.len(), 2);
14543        for candidate in &runner.state.candidates {
14544            assert_eq!(
14545                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14546                git::tree_of(&repo, &seed).await.unwrap()
14547            );
14548            assert_eq!(
14549                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14550                "required commit"
14551            );
14552        }
14553        assert!(runner.state.worktree_root().join("judge-1").exists());
14554        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14555        runner.prep().await.unwrap();
14556        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14557
14558        runner.state.prep_complete = Some(false);
14559        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14560            run: Some(format!(
14561                "test -f '{}'",
14562                tmp.path().join("dependency-ready").display()
14563            )),
14564            ..Default::default()
14565        }];
14566        assert!(runner.prep().await.is_err());
14567        assert!(runner.state.prep_failure.is_some());
14568        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14569        runner.state = RunState::load(&runner.state.id).unwrap();
14570        assert!(
14571            runner.prep().await.is_err(),
14572            "correcting the cause alone must not auto-retry"
14573        );
14574        let failed_at = runner.state.events.last().unwrap().at;
14575        assert!(
14576            runner.retry_preparation(failed_at).is_err(),
14577            "old approvals cannot unlock the hold"
14578        );
14579        let mut task = crate::queue::Task::new(
14580            "retry".into(),
14581            "retry setup".into(),
14582            repo.clone(),
14583            crate::queue::Source::Human,
14584        );
14585        task.start(runner.state.id.clone());
14586        task.hold_machine(Some("setup failed".into()));
14587        task.release_operator("cli", None);
14588        runner
14589            .retry_preparation(task.resumption.as_ref().unwrap().at)
14590            .unwrap();
14591        runner.prep().await.unwrap();
14592        assert!(runner.state.prep_failure.is_none());
14593        assert_eq!(runner.state.prep_complete, Some(true));
14594        assert_eq!(runner.state.candidates.len(), 2);
14595        assert!(runner.state.worktree_root().join("judge-1").exists());
14596        for candidate in &runner.state.candidates {
14597            assert_eq!(
14598                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14599                git::tree_of(&repo, &seed).await.unwrap()
14600            );
14601        }
14602    }
14603
14604    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14605        let mut heads = Vec::new();
14606        for candidate in candidates {
14607            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14608        }
14609        heads
14610    }
14611
14612    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14613        ask_test_home();
14614        let tmp = tempfile::tempdir().unwrap();
14615        let repo = tmp.path().join("repo");
14616        std::fs::create_dir(&repo).unwrap();
14617        init_repo(&repo);
14618        let mut runner = runner_at(RunStatus::Reviewing);
14619        runner.state.repo = repo.clone();
14620        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14621        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14622        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14623            let branch = format!("candidate-{label}");
14624            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14625            git::worktree_add_branch(&repo, &worktree, &branch, "main")
14626                .await
14627                .unwrap();
14628            if label == 'B' {
14629                std::fs::write(worktree.join("change"), "winner").unwrap();
14630                git::rescue_commit(&worktree, "candidate change")
14631                    .await
14632                    .unwrap();
14633            }
14634            runner.state.candidates.push(Candidate {
14635                index,
14636                label,
14637                agent: "alpha".into(),
14638                branch,
14639                worktree,
14640                summary: "completed reply".into(),
14641                stat: String::new(),
14642                files: usize::from(label == 'B'),
14643                commits: usize::from(label == 'B'),
14644                empty: label == 'A',
14645                failed: None,
14646                verified_noop: None,
14647                duration_ms: 1,
14648                folded: false,
14649            });
14650        }
14651        runner.state.judge_skipped = true;
14652        runner.state.tally = Some(Tally {
14653            first_choice: BTreeMap::new(),
14654            borda: BTreeMap::new(),
14655            winner: 'B',
14656            rankings: 0,
14657            unanimous_initial: false,
14658            deliberated: false,
14659            changed_votes: 0,
14660            unanimous_final: false,
14661            tie_break: None,
14662            judges: 0,
14663            present: 0,
14664            quorum: 0,
14665            met_quorum: true,
14666            uncontested: Some("only B produced a change".into()),
14667        });
14668        runner.state.save().unwrap();
14669        (tmp, runner)
14670    }
14671
14672    #[tokio::test]
14673    #[cfg(unix)]
14674    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14675        for (solo, failed_checkpoint) in
14676            [(true, false), (false, false), (true, true), (false, true)]
14677        {
14678            let (_tmp, mut runner) = preservation_resume_fixture().await;
14679            if solo {
14680                runner.state.candidates.retain(|c| c.label == 'A');
14681            }
14682            runner.state.candidates[0].empty = false;
14683            runner.state.candidates[0].summary.clear();
14684            runner.state.tally = None;
14685            runner.state.judge_skipped = false;
14686            runner.state.status = RunStatus::Implementing;
14687            runner.state.config.disk.min_free_bytes = u64::MAX;
14688            let mut implementer = spec("alpha");
14689            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14690            runner.roles.implementers = vec![implementer];
14691            let patch = runner.state.dir().join("artifacts/cand-A.patch");
14692            if failed_checkpoint {
14693                std::fs::create_dir_all(&patch).unwrap();
14694            }
14695            runner.state.save().unwrap();
14696            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14697            runner.state = RunState::load(&runner.state.id).unwrap();
14698            assert_eq!(runner.state.status, RunStatus::Blocked);
14699            assert!(runner.state.interrupted_implementations.contains(&'A'));
14700            assert_eq!(runner.state.candidates[0].commits, 0);
14701            assert!(!runner.state.candidates[0].empty);
14702            assert!(runner.state.candidates[0].failed.is_none());
14703            if failed_checkpoint {
14704                assert!(runner.state.candidate_preservation.contains_key(&'A'));
14705                std::fs::remove_dir(patch).unwrap();
14706            }
14707            runner.state.resource_block = None;
14708            runner.state.config.disk.min_free_bytes = 0;
14709            runner.roles.implementers[0].command = vec![
14710                "sh".into(),
14711                "-c".into(),
14712                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14713            ];
14714            runner.implement().await.unwrap();
14715            assert!(runner.state.interrupted_implementations.is_empty());
14716            assert!(runner.state.candidate_preservation.is_empty());
14717            assert_eq!(runner.state.candidates[0].commits, 1);
14718            assert!(!runner.state.candidates[0].empty);
14719            assert!(
14720                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14721                    .unwrap()
14722                    .contains("retried empty interrupted seat")
14723            );
14724            if !solo {
14725                assert_eq!(runner.state.candidates[1].commits, 1);
14726            }
14727        }
14728    }
14729
14730    #[tokio::test]
14731    #[cfg(unix)]
14732    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14733        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14734            let (_tmp, mut runner) = preservation_resume_fixture().await;
14735            runner.state.candidates.retain(|c| c.label == 'A');
14736            runner.state.candidates[0].empty = false;
14737            runner.state.candidates[0].summary.clear();
14738            runner.state.tally = None;
14739            runner.state.judge_skipped = false;
14740            runner.state.status = RunStatus::Implementing;
14741            // Force the real post-spawn watchdog, independently of available
14742            // disk space. Dirty/committed work predates its first measurement.
14743            runner.state.config.disk.min_free_bytes = u64::MAX;
14744            let wt = runner.state.candidates[0].worktree.clone();
14745            std::fs::write(
14746                wt.join("interrupted-source"),
14747                "retain implementation work\n",
14748            )
14749            .unwrap();
14750            if committed {
14751                git::rescue_commit(&wt, "implementation checkpoint")
14752                    .await
14753                    .unwrap();
14754            }
14755            let lock = git::git(
14756                &wt,
14757                &[
14758                    "rev-parse",
14759                    "--path-format=absolute",
14760                    "--git-path",
14761                    "index.lock",
14762                ],
14763            )
14764            .await
14765            .unwrap();
14766            if block_rescue {
14767                std::fs::write(&lock, "fixture-owned lock").unwrap();
14768            }
14769            let mut implementer = spec("alpha");
14770            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14771            runner.roles.implementers = vec![implementer];
14772            runner.state.save().unwrap();
14773            let result = runner.implement().await;
14774            assert_eq!(result.is_err(), block_rescue);
14775            assert_eq!(
14776                runner.state.resource_block.as_ref().unwrap().kind,
14777                "disk pressure"
14778            );
14779            assert!(
14780                runner
14781                    .state
14782                    .events
14783                    .iter()
14784                    .any(|e| e.message.contains("agent execution interrupted"))
14785            );
14786            assert!(
14787                runner
14788                    .state
14789                    .dir()
14790                    .join("artifacts/candidate-A-recovery/staged.patch")
14791                    .exists()
14792            );
14793            if block_rescue {
14794                let saved = RunState::load(&runner.state.id).unwrap();
14795                assert!(saved.candidate_preservation.contains_key(&'A'));
14796                assert!(!saved.candidates[0].empty);
14797                assert_eq!(
14798                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14799                    "retain implementation work\n"
14800                );
14801                std::fs::remove_file(lock).unwrap();
14802                runner.state = saved;
14803            } else {
14804                assert_eq!(runner.state.status, RunStatus::Blocked);
14805                assert_eq!(runner.state.candidates[0].commits, 1);
14806            }
14807            // Recovery must save the candidate before deciding whether another
14808            // implementation is needed. No implementer is available on retry.
14809            runner.roles.implementers.clear();
14810            runner.state.config.disk.min_free_bytes = 0;
14811            runner.state.resource_block = None;
14812            runner.implement().await.unwrap();
14813            assert!(runner.state.candidate_preservation.is_empty());
14814            assert_eq!(runner.state.candidates[0].commits, 1);
14815            assert!(!runner.state.candidates[0].empty);
14816            assert!(git::is_clean(&wt).await.unwrap());
14817            assert!(
14818                crate::run::read_artifact(&runner.state, "cand-A.patch")
14819                    .unwrap()
14820                    .contains("retain implementation work")
14821            );
14822        }
14823    }
14824
14825    #[tokio::test]
14826    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14827        let (_tmp, mut runner) = preservation_resume_fixture().await;
14828        // Schema 15 already confirmed the unfolded no-change result.
14829        runner.implement().await.unwrap();
14830        assert!(runner.state.candidate_preservation.is_empty());
14831        assert!(
14832            !runner
14833                .state
14834                .dir()
14835                .join("artifacts/candidate-A-recovery")
14836                .exists()
14837        );
14838        runner.fold_losers().await.unwrap();
14839        assert!(!runner.state.candidates[0].worktree.exists());
14840        for schema in [14, crate::run::SCHEMA] {
14841            runner.state.schema = schema;
14842            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14843            runner.state.save().unwrap();
14844            runner.state = RunState::load(&runner.state.id).unwrap();
14845            runner.implement().await.unwrap();
14846            assert!(runner.state.candidate_preservation.is_empty());
14847            assert!(runner.state.candidates[0].folded);
14848            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14849        }
14850    }
14851
14852    #[tokio::test]
14853    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
14854        let (_tmp, mut runner) = preservation_resume_fixture().await;
14855        runner.state.candidates.retain(|c| c.label == 'A');
14856        runner.state.tally = None;
14857        runner.state.judge_skipped = false;
14858        let candidate = &mut runner.state.candidates[0];
14859        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
14860        candidate.failed = Some("deployment validation unavailable".into());
14861        std::fs::write(
14862            candidate.worktree.join("repair"),
14863            "future launcher policy\n",
14864        )
14865        .unwrap();
14866        runner
14867            .state
14868            .candidate_preservation
14869            .insert('A', "awaiting persistence".into());
14870        runner.persist_candidates().await.unwrap();
14871        let candidate = &runner.state.candidates[0];
14872        assert!(!candidate.empty);
14873        assert_eq!(candidate.commits, 1);
14874        assert_eq!(candidate.files, 1);
14875        assert!(candidate.failed.is_none());
14876        assert!(candidate.summary.contains("EPERM"));
14877        assert!(
14878            crate::run::read_artifact(&runner.state, "cand-A.patch")
14879                .unwrap()
14880                .contains("future launcher policy")
14881        );
14882        assert!(runner.state.candidate_preservation.is_empty());
14883    }
14884
14885    #[tokio::test]
14886    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14887        let (_tmp, mut runner) = preservation_resume_fixture().await;
14888        let wt = runner.state.candidates[0].worktree.clone();
14889        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14890        runner.state.schema = 14;
14891        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14892        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14893        let tally = serde_json::to_value(&runner.state.tally).unwrap();
14894        for _ in 0..2 {
14895            assert!(runner.implement().await.is_err());
14896            assert_eq!(runner.state.status, RunStatus::Blocked);
14897            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14898            assert!(
14899                runner.state.candidates[0].empty,
14900                "old verdict must remain unusable"
14901            );
14902            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14903            assert!(runner.state.judge_skipped);
14904            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14905            assert!(
14906                crate::run::read_artifact(&runner.state, "cand-A.patch")
14907                    .unwrap()
14908                    .contains("owned edits")
14909            );
14910            assert!(wt.exists());
14911            assert_eq!(
14912                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14913                    .await
14914                    .unwrap(),
14915                1
14916            );
14917            runner.state = RunState::load(&runner.state.id).unwrap();
14918        }
14919    }
14920
14921    #[tokio::test]
14922    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14923        let (_tmp, mut runner) = preservation_resume_fixture().await;
14924        let wt = runner.state.candidates[0].worktree.clone();
14925        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14926        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14927        let index_lock = PathBuf::from(
14928            git::git(
14929                &wt,
14930                &[
14931                    "rev-parse",
14932                    "--path-format=absolute",
14933                    "--git-path",
14934                    "index.lock",
14935                ],
14936            )
14937            .await
14938            .unwrap(),
14939        );
14940        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14941        runner.state.schema = 14;
14942        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14943        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14944        assert!(runner.implement().await.is_err());
14945        assert!(runner.after_implement().is_err());
14946        assert!(runner.fold_losers().await.is_err());
14947        assert_eq!(
14948            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14949            "failed seat edits"
14950        );
14951        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14952        runner.state = RunState::load(&runner.state.id).unwrap();
14953        std::fs::remove_file(index_lock).unwrap();
14954        // Recovery finds changes that the old uncontested result never judged.
14955        assert!(runner.implement().await.is_err());
14956        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14957        assert!(runner.state.candidates[0].empty);
14958        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14959        assert!(
14960            crate::run::read_artifact(&runner.state, "cand-A.patch")
14961                .unwrap()
14962                .contains("failed seat edits")
14963        );
14964        assert!(wt.exists());
14965    }
14966
14967    #[tokio::test]
14968    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14969        let (_tmp, mut runner) = preservation_resume_fixture().await;
14970        let repo = runner.state.repo.clone();
14971        let common = git::git(
14972            &repo,
14973            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14974        )
14975        .await
14976        .unwrap();
14977        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14978        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14979        assert!(runner.fold_losers().await.is_err());
14980        assert!(!runner.state.candidates[0].worktree.exists());
14981        assert!(!runner.state.candidates[0].folded);
14982        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14983        runner.state = RunState::load(&runner.state.id).unwrap();
14984        std::fs::remove_file(branch_lock).unwrap();
14985        let judge = runner.state.worktree_root().join("judge-1");
14986        git::worktree_add_detached(&repo, &judge, "main")
14987            .await
14988            .unwrap();
14989        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14990        runner.roles.judges = vec![spec("judge")];
14991        assert!(runner.fold_losers().await.is_err());
14992        runner.state = RunState::load(&runner.state.id).unwrap();
14993        assert!(
14994            runner.state.candidates[0].folded,
14995            "candidate progress saved before judge failure"
14996        );
14997        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14998        assert!(judge.join("uncommitted").exists());
14999        std::fs::remove_file(judge.join("uncommitted")).unwrap();
15000        runner.fold_losers().await.unwrap();
15001        assert!(!judge.exists());
15002        runner.fold_losers().await.unwrap();
15003    }
15004
15005    #[tokio::test]
15006    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
15007        let (_tmp, mut runner) = preservation_resume_fixture().await;
15008        runner.state.candidates.retain(|c| c.label == 'B');
15009        runner.state.tally = None;
15010        runner.state.judge_skipped = false;
15011        runner.state.prep_complete = Some(true);
15012        runner.state.config.graph.advise = false;
15013        runner.state.config.graph.selected_implementer = Some("alpha".into());
15014        let wt = runner.state.candidates[0].worktree.clone();
15015        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
15016        runner
15017            .state
15018            .candidate_preservation
15019            .insert('B', "awaiting persistence".into());
15020        let index_lock = git::git(
15021            &wt,
15022            &[
15023                "rev-parse",
15024                "--path-format=absolute",
15025                "--git-path",
15026                "index.lock",
15027            ],
15028        )
15029        .await
15030        .unwrap();
15031        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15032        runner.state.save().unwrap();
15033        assert!(runner.execute_graph().await.is_err());
15034        assert_eq!(runner.state.status, RunStatus::Blocked);
15035        assert!(runner.state.candidate_preservation.contains_key(&'B'));
15036        std::fs::remove_file(index_lock).unwrap();
15037        // Park at the next graph boundary: this tests the real selected-seat
15038        // hold guard without dispatching reviewers or publishing anything.
15039        for _ in 0..2 {
15040            runner.state = RunState::load(&runner.state.id).unwrap();
15041            runner.pause.park();
15042            runner.execute_graph().await.unwrap();
15043            assert_eq!(runner.state.status, RunStatus::Implementing);
15044            assert!(runner.state.candidate_preservation.is_empty());
15045            assert_eq!(runner.state.candidates[0].commits, 2);
15046            assert!(runner.state.tally.is_none());
15047        }
15048        runner.implement().await.unwrap();
15049        assert_eq!(runner.state.status, RunStatus::Judging);
15050        // A genuine selected-implementer hold must still require release.
15051        runner.state.status = RunStatus::Blocked;
15052        runner.state.candidates[0].failed = Some("implementation failed".into());
15053        runner.state.save().unwrap();
15054        let error = runner.execute_graph().await.unwrap_err();
15055        assert!(
15056            error
15057                .to_string()
15058                .contains("selected implementation is held")
15059        );
15060    }
15061
15062    #[tokio::test]
15063    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
15064        let (_tmp, mut runner) = preservation_resume_fixture().await;
15065        let repo = runner.state.repo.clone();
15066        let candidate = runner.state.candidates[0].clone();
15067        // Remove only the disposable fixture directory, leaving Git's linked
15068        // worktree registration and branch intact, as in a vanished checkout.
15069        let registered = git::worktree_holding(&repo, &candidate.branch)
15070            .await
15071            .unwrap()
15072            .unwrap();
15073        assert_eq!(
15074            registered.canonicalize().unwrap(),
15075            candidate.worktree.canonicalize().unwrap()
15076        );
15077        std::fs::remove_dir_all(&candidate.worktree).unwrap();
15078        assert_eq!(
15079            git::worktree_holding(&repo, &candidate.branch)
15080                .await
15081                .unwrap(),
15082            Some(registered)
15083        );
15084        assert!(
15085            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
15086            "the stale registration initially keeps the branch checked out"
15087        );
15088        runner.fold_losers().await.unwrap();
15089        assert!(runner.state.candidates[0].folded);
15090        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15091        assert_eq!(
15092            git::worktree_holding(&repo, &candidate.branch)
15093                .await
15094                .unwrap(),
15095            None
15096        );
15097        runner.state = RunState::load(&runner.state.id).unwrap();
15098        runner.fold_losers().await.unwrap();
15099    }
15100
15101    #[tokio::test]
15102    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
15103        let (_tmp, mut runner) = preservation_resume_fixture().await;
15104        let repo = runner.state.repo.clone();
15105        let candidate = runner.state.candidates[0].clone();
15106        let winner = runner.state.candidates[1].clone();
15107        let registered = git::worktree_holding(&repo, &candidate.branch)
15108            .await
15109            .unwrap()
15110            .unwrap();
15111        assert_eq!(
15112            registered.canonicalize().unwrap(),
15113            candidate.worktree.canonicalize().unwrap()
15114        );
15115        std::fs::remove_dir_all(&candidate.worktree).unwrap();
15116        assert_eq!(
15117            git::worktree_holding(&repo, &candidate.branch)
15118                .await
15119                .unwrap(),
15120            Some(registered)
15121        );
15122        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
15123
15124        for _ in 0..2 {
15125            fold_run(&mut runner.state, false, &crate::run::home())
15126                .await
15127                .unwrap();
15128            runner.state = RunState::load(&runner.state.id).unwrap();
15129            assert!(runner.state.candidates[0].folded);
15130            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15131            assert_eq!(
15132                git::worktree_holding(&repo, &candidate.branch)
15133                    .await
15134                    .unwrap(),
15135                None
15136            );
15137            assert!(!runner.state.candidates[1].folded);
15138            assert!(winner.worktree.exists());
15139            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
15140        }
15141    }
15142
15143    #[tokio::test]
15144    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
15145        ask_test_home();
15146        let tmp = tempfile::tempdir().unwrap();
15147        let repo = tmp.path().join("repo");
15148        std::fs::create_dir(&repo).unwrap();
15149        init_repo(&repo);
15150        let mut runner = runner_at(RunStatus::Implementing);
15151        runner.state.repo = repo.clone();
15152        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15153        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15154        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15155            let branch = format!("candidate-{label}");
15156            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
15157            git::worktree_add_branch(&repo, &wt, &branch, "main")
15158                .await
15159                .unwrap();
15160            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
15161            git::git(&wt, &["add", "README.md"]).await.unwrap();
15162            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
15163            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
15164            runner.state.candidates.push(Candidate {
15165                index,
15166                label,
15167                agent: "alpha".to_owned(),
15168                branch,
15169                worktree: wt,
15170                summary: "completed implementation".to_owned(),
15171                stat: String::new(),
15172                files: 0,
15173                commits: 0,
15174                empty: false,
15175                failed: None,
15176                verified_noop: None,
15177                duration_ms: 42,
15178                folded: false,
15179            });
15180            runner
15181                .state
15182                .candidate_preservation
15183                .insert(label, "awaiting persistence".to_owned());
15184        }
15185        let wt = runner.state.candidates[0].worktree.clone();
15186        let index_lock = git::git(
15187            &wt,
15188            &[
15189                "rev-parse",
15190                "--path-format=absolute",
15191                "--git-path",
15192                "index.lock",
15193            ],
15194        )
15195        .await
15196        .unwrap();
15197        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15198        runner.state.save().unwrap();
15199        assert!(runner.persist_candidates().await.is_err());
15200        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15201        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
15202        assert_eq!(
15203            runner.state.candidates[1].commits, 1,
15204            "other results are still collected"
15205        );
15206        assert!(
15207            !runner.state.candidates[0].empty,
15208            "failed rescue is not no change"
15209        );
15210        assert_eq!(
15211            runner.state.candidates[0].summary,
15212            "completed implementation"
15213        );
15214        assert!(runner.after_implement().is_err());
15215        let home = tmp.path().join("fold-home");
15216        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
15217        assert!(wt.join("new-file").exists());
15218        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
15219        assert!(recovery.join("staged.patch").exists());
15220        assert!(recovery.join("unstaged.patch").exists());
15221        assert_eq!(
15222            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
15223            "untracked edits\n"
15224        );
15225        runner.state.begin_pass();
15226        runner.state.end_pass_parked(ParkCause::Upgrade {
15227            from: "1.0".into(),
15228            to: Some("1.1".into()),
15229        });
15230        runner.state.save().unwrap();
15231        runner.state = RunState::load(&runner.state.id).unwrap();
15232        assert!(matches!(
15233            runner.state.passes.last().unwrap().end,
15234            Some(crate::run::PassEnd::Parked {
15235                cause: ParkCause::Upgrade { .. }
15236            })
15237        ));
15238        std::fs::remove_file(index_lock).unwrap();
15239        // No implementer is configured: a persistence retry cannot dispatch a seat.
15240        runner.implement().await.unwrap();
15241        assert!(runner.state.candidate_preservation.is_empty());
15242        assert!(
15243            runner
15244                .state
15245                .candidates
15246                .iter()
15247                .all(|c| c.commits == 1 && !c.empty)
15248        );
15249        assert!(git::is_clean(&wt).await.unwrap());
15250        assert!(
15251            crate::run::read_artifact(&runner.state, "cand-A.patch")
15252                .unwrap()
15253                .contains("later tracked edits")
15254        );
15255    }
15256
15257    #[tokio::test]
15258    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
15259        ask_test_home();
15260        let tmp = tempfile::tempdir().unwrap();
15261        let repo = tmp.path().join("repo");
15262        std::fs::create_dir(&repo).unwrap();
15263        init_repo(&repo);
15264        let wt = tmp.path().join("candidate");
15265        git::worktree_add_branch(&repo, &wt, "candidate", "main")
15266            .await
15267            .unwrap();
15268        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15269        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
15270        assert!(wt.join("uncommitted").exists());
15271        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15272        std::fs::remove_file(wt.join("uncommitted")).unwrap();
15273        let mut runner = runner_at(RunStatus::Implementing);
15274        runner.state.repo = repo.clone();
15275        runner.state.base_commit = "missing-base".to_owned();
15276        runner.state.candidates.push(Candidate {
15277            index: 0,
15278            label: 'A',
15279            agent: "alpha".into(),
15280            branch: "candidate".into(),
15281            worktree: wt.clone(),
15282            summary: "completed".into(),
15283            stat: String::new(),
15284            files: 0,
15285            commits: 0,
15286            empty: false,
15287            failed: None,
15288            verified_noop: None,
15289            duration_ms: 1,
15290            folded: false,
15291        });
15292        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15293        assert!(
15294            fold_run(&mut runner.state, true, &tmp.path().join("home"))
15295                .await
15296                .is_err()
15297        );
15298        assert!(!runner.state.candidates[0].folded);
15299        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15300        std::fs::remove_file(wt.join("uncommitted")).unwrap();
15301        runner
15302            .state
15303            .candidate_preservation
15304            .insert('A', "pending".into());
15305        assert!(runner.persist_candidates().await.is_err());
15306        assert!(!runner.state.candidates[0].empty);
15307        assert!(!runner.state.candidates[0].folded);
15308        assert!(wt.exists());
15309        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15310        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
15311        std::fs::create_dir_all(&patch_path).unwrap();
15312        assert!(
15313            runner.persist_candidates().await.is_err(),
15314            "artifact failure stays pending"
15315        );
15316        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15317        assert!(!runner.state.candidates[0].empty);
15318        std::fs::remove_dir(patch_path).unwrap();
15319        runner.persist_candidates().await.unwrap();
15320        assert!(
15321            runner.state.candidates[0].empty,
15322            "only successful inspection confirms no change"
15323        );
15324    }
15325
15326    fn init_repo(dir: &Path) {
15327        let run = |args: &[&str]| {
15328            let out = std::process::Command::new("git")
15329                .args(args)
15330                .current_dir(dir)
15331                .quiet()
15332                .output()
15333                .expect("spawn git");
15334            assert!(
15335                out.status.success(),
15336                "git {args:?} failed: {}",
15337                String::from_utf8_lossy(&out.stderr)
15338            );
15339        };
15340        run(&["init", "-b", "main"]);
15341        run(&["config", "user.name", "magi test"]);
15342        run(&["config", "user.email", "magi@example.com"]);
15343        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
15344        run(&["add", "-A"]);
15345        run(&["commit", "-m", "init"]);
15346    }
15347
15348    // `settle_questions` is what closes the ghost the phone showed: a run's
15349    // seat asked something, the run then ended, and nothing was left to
15350    // abandon the question it left `open`. `HOME` is a process-wide
15351    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
15352    // first time it runs in the binary — every test below still reaches the
15353    // same directory whichever call won, and each gets its own run id from
15354    // `RunState::new`, so they never collide there.
15355    fn ask_test_home() {
15356        crate::run::pin_test_home();
15357    }
15358
15359    #[cfg(unix)]
15360    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
15361        ask_test_home();
15362        let tmp = tempfile::tempdir().unwrap();
15363        let repo = tmp.path().join("repo");
15364        std::fs::create_dir(&repo).unwrap();
15365        init_repo(&repo);
15366        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15367        let origin = tmp.path().join("origin.git");
15368        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
15369            .await
15370            .unwrap();
15371        git::git(
15372            &repo,
15373            &["remote", "add", "origin", origin.to_str().unwrap()],
15374        )
15375        .await
15376        .unwrap();
15377        git::git(&repo, &["checkout", "-b", "fix/retained"])
15378            .await
15379            .unwrap();
15380        std::fs::write(repo.join("retained"), "retained change").unwrap();
15381        git::git(&repo, &["add", "."]).await.unwrap();
15382        git::git(&repo, &["commit", "-m", "retained implementation"])
15383            .await
15384            .unwrap();
15385        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15386        git::git(&repo, &["checkout", "main"]).await.unwrap();
15387        std::fs::write(repo.join("base-change"), "new base").unwrap();
15388        git::git(&repo, &["add", "."]).await.unwrap();
15389        git::git(&repo, &["commit", "-m", "base moved"])
15390            .await
15391            .unwrap();
15392        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15393        git::git(&repo, &["checkout", "fix/retained"])
15394            .await
15395            .unwrap();
15396        let mut runner = runner_at(RunStatus::Voting);
15397        runner.state.repo = repo.clone();
15398        runner.state.base_commit = base;
15399        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15400        runner.state.config.disk.min_free_bytes = 0;
15401        runner.state.config.graph.review_rounds = 1;
15402        runner.state.config.verify.e2e = vec!["true".into()];
15403        runner.state.config.verify.gate = vec!["true".into()];
15404        runner.state.candidates = vec![Candidate {
15405            index: 0,
15406            label: 'A',
15407            agent: "fixture".into(),
15408            branch: "fix/retained".into(),
15409            worktree: repo.clone(),
15410            summary: String::new(),
15411            stat: String::new(),
15412            files: 1,
15413            commits: 1,
15414            empty: false,
15415            failed: None,
15416            verified_noop: None,
15417            duration_ms: 0,
15418            folded: false,
15419        }];
15420        runner.state.tally = Some(Tally {
15421            first_choice: BTreeMap::new(),
15422            borda: BTreeMap::new(),
15423            winner: 'A',
15424            rankings: 0,
15425            unanimous_initial: true,
15426            deliberated: false,
15427            changed_votes: 0,
15428            unanimous_final: true,
15429            tie_break: None,
15430            judges: 0,
15431            present: 0,
15432            quorum: 0,
15433            met_quorum: true,
15434            uncontested: Some("review-only".into()),
15435        });
15436        let mut task = crate::queue::Task::new(
15437            "fixture".into(),
15438            "fixture".into(),
15439            repo,
15440            crate::queue::Source::Human,
15441        );
15442        task.link_run(&runner.state.id);
15443        crate::queue::Queue::open().put(&mut task).unwrap();
15444        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
15445        runner.state.base_sync = Some(BaseSync {
15446            tip: String::new(),
15447            behind: 1,
15448            attempts: BASE_SYNC_ROUNDS,
15449            conflict: None,
15450            already_in: None,
15451        });
15452        runner.sync_to_base().await.unwrap();
15453        assert_eq!(runner.state.status, RunStatus::Blocked);
15454        (tmp, runner, head, task.id)
15455    }
15456
15457    #[cfg(unix)]
15458    #[tokio::test]
15459    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
15460        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
15461        if std::env::var_os(CHILD).is_none() {
15462            let home = tempfile::tempdir().unwrap();
15463            let output = std::process::Command::new(std::env::current_exe().unwrap())
15464                .args([
15465                    "--exact",
15466                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
15467                    "--nocapture",
15468                ])
15469                .env(CHILD, "1")
15470                .env("MAGI_HOME", home.path())
15471                .env("MAGI_WORKER_HOME", home.path())
15472                .env("MAGI_WORKER_ID", "abcdef")
15473                .output()
15474                .unwrap();
15475            assert!(
15476                output.status.success()
15477                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15478                "{}\n{}",
15479                String::from_utf8_lossy(&output.stdout),
15480                String::from_utf8_lossy(&output.stderr)
15481            );
15482            return;
15483        }
15484        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
15485        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15486        let home = crate::run::home();
15487        let queue = crate::queue::Queue::open();
15488        let mut owner = queue.get(&task).unwrap();
15489        owner.release_operator("test-release", None);
15490        owner.start(runner.state.id.clone());
15491        queue.put(&mut owner).unwrap();
15492        runner.state.driver_exited = true;
15493        runner.state.task_instruction = Some(owner.instruction.clone());
15494        runner.state.save().unwrap();
15495        let _claim = queue.claim(&task).unwrap();
15496        let reserve = |id: &str| {
15497            let directory = home.join("workers/jobs").join(id);
15498            std::fs::create_dir_all(&directory).unwrap();
15499            let job = crate::worker::Job {
15500                id: id.into(),
15501                sequence: 1,
15502                kind: "task".into(),
15503                key: format!("task:{task}"),
15504                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
15505                generation: "fixture-generation".into(),
15506                version: None,
15507                status: crate::worker::Status::Running,
15508                result: None,
15509                pid: Some(std::process::id()),
15510                created_at: 0,
15511                finished_at: None,
15512            };
15513            std::fs::write(
15514                directory.join("job.json"),
15515                serde_json::to_vec(&job).unwrap(),
15516            )
15517            .unwrap();
15518            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
15519        };
15520        let _own = reserve("abcdef");
15521        let other = reserve("fedcba");
15522        assert!(
15523            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15524                .await
15525                .is_err()
15526        );
15527        assert!(
15528            RunState::load(&runner.state.id)
15529                .unwrap()
15530                .base_sync_recovery
15531                .authorizations
15532                .is_empty()
15533        );
15534        drop(other);
15535        assert!(crate::daemon::is_working_on(
15536            &home,
15537            &runner.state.id,
15538            Timestamp::now()
15539        ));
15540        assert!(
15541            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
15542        );
15543        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15544            .await
15545            .unwrap();
15546        assert!(
15547            RunState::load(&runner.state.id)
15548                .unwrap()
15549                .base_recovery_pending()
15550        );
15551    }
15552
15553    #[cfg(unix)]
15554    #[tokio::test]
15555    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
15556        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15557        let home = crate::run::home();
15558        runner.state.driver_pid = Some(std::process::id());
15559        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
15560        runner.state.driver_exited = false;
15561        runner.state.begin_pass();
15562        runner.state.save().unwrap();
15563        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
15564        driver.record(&runner.state).unwrap();
15565        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
15566        // Preserve the durable claim as if the process died before Drop saved.
15567        drop(driver);
15568        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
15569        assert_eq!(
15570            runner.state.liveness_under(&home, false),
15571            crate::run::Liveness::Dead
15572        );
15573        let queue = crate::queue::Queue::open();
15574        let _claim = queue.claim(&task).unwrap();
15575        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15576            .await
15577            .unwrap();
15578        let disk = RunState::load(&runner.state.id).unwrap();
15579        assert!(!disk.driver_exited);
15580        assert!(disk.base_recovery_pending());
15581    }
15582
15583    #[cfg(unix)]
15584    #[tokio::test]
15585    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
15586        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
15587        if std::env::var_os(CHILD).is_none() {
15588            use std::os::unix::fs::PermissionsExt;
15589            let tmp = tempfile::tempdir().unwrap();
15590            let gh = tmp.path().join("gh");
15591            std::fs::write(&gh, r#"#!/bin/sh
15592case "$1 $2" in
15593  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
15594  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
15595  *) exit 2 ;;
15596esac
15597"#).unwrap();
15598            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
15599            let mut paths = vec![tmp.path().to_path_buf()];
15600            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
15601            let output = std::process::Command::new(std::env::current_exe().unwrap())
15602                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
15603                .env(CHILD, "1")
15604                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
15605                .env("PATH", std::env::join_paths(paths).unwrap())
15606                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
15607                .output().unwrap();
15608            assert!(
15609                output.status.success()
15610                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15611                "{}\n{}",
15612                String::from_utf8_lossy(&output.stdout),
15613                String::from_utf8_lossy(&output.stderr)
15614            );
15615            return;
15616        }
15617        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15618        let repo = runner.state.repo.clone();
15619        git::git(&repo, &["checkout", "main"]).await.unwrap();
15620        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
15621        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15622        git::git(&repo, &["checkout", "fix/retained"])
15623            .await
15624            .unwrap();
15625        runner.state.pr = Some(crate::run::PrRecord {
15626            url: "https://example.test/project/pull/1".into(),
15627            number: 1,
15628            state: "open".into(),
15629            checks: "unknown".into(),
15630            round: 0,
15631            rounds: 1,
15632            red_at_merge: Vec::new(),
15633        });
15634        runner.sync_to_base().await.unwrap();
15635        assert_eq!(runner.state.status, RunStatus::Blocked);
15636        assert_eq!(
15637            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
15638            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
15639        );
15640        assert!(runner.state.base_sync_closure_pending());
15641        assert!(crate::handover::can_resume(&runner.state));
15642        let queue = crate::queue::Queue::open();
15643        assert!(!crate::base_sync_recovery::held_task(
15644            &queue,
15645            &queue.get(&task).unwrap()
15646        ));
15647        assert!(runner.state.authorize_base_sync(&task).is_err());
15648        // Legacy closure stops have no typed stop but must retain the checkpoint.
15649        runner.state.base_sync_recovery.stop = None;
15650        runner.state.status = RunStatus::Voting;
15651        runner.state.gate_ran = true;
15652        runner.state.config.graph.review_rounds = 0;
15653        runner.state.save().unwrap();
15654        assert!(crate::handover::can_resume(&runner.state));
15655        runner.state = RunState::load(&runner.state.id).unwrap();
15656        runner.execute_graph().await.unwrap();
15657        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
15658        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
15659        assert_eq!(
15660            runner.state.base_sync.as_ref().unwrap().attempts,
15661            BASE_SYNC_ROUNDS
15662        );
15663        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
15664        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15665        assert!(runner.state.reviews.is_empty());
15666        assert!(runner.state.gate.is_empty());
15667    }
15668
15669    #[cfg(unix)]
15670    #[tokio::test]
15671    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
15672        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15673        runner.state.driver_exited = true;
15674        runner.state.save().unwrap();
15675        let home = crate::run::home();
15676        let queue = crate::queue::Queue::open();
15677        let _claim = queue.claim(&task).unwrap();
15678        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
15679        assert!(
15680            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15681                .await
15682                .is_err()
15683        );
15684        drop(lease);
15685        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
15686        assert!(
15687            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15688                .await
15689                .is_err()
15690        );
15691        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
15692        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15693            .await
15694            .unwrap();
15695        let disk = RunState::load(&runner.state.id).unwrap();
15696        assert!(disk.base_recovery_pending());
15697        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
15698        assert!(
15699            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15700                .await
15701                .is_err()
15702        );
15703        assert_eq!(
15704            RunState::load(&runner.state.id)
15705                .unwrap()
15706                .base_sync_recovery
15707                .authorizations
15708                .len(),
15709            1
15710        );
15711        let mut owner = queue.get(&task).unwrap();
15712        owner.runs.push("20260101-000000-ffff".into());
15713        queue.put(&mut owner).unwrap();
15714        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15715            .await
15716            .unwrap_err();
15717        assert!(error.to_string().contains("run task ownership changed"));
15718    }
15719
15720    #[cfg(unix)]
15721    #[tokio::test]
15722    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
15723        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
15724        let mut legacy = serde_json::to_value(&runner.state).unwrap();
15725        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
15726        legacy["status"] = serde_json::json!("voting");
15727        runner.state = serde_json::from_value(legacy).unwrap();
15728        runner.state.save().unwrap();
15729        for _ in 0..3 {
15730            runner.execute_graph().await.unwrap();
15731            assert_eq!(runner.state.status, RunStatus::Blocked);
15732            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
15733            assert!(!crate::handover::can_resume(&runner.state));
15734            assert_eq!(
15735                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15736                head
15737            );
15738            assert!(runner.state.reviews.is_empty());
15739            assert!(!runner.state.gate_ran);
15740        }
15741        let notice = crate::notices::run_ended(&runner.state).unwrap();
15742        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
15743        runner.state.begin_pass();
15744        runner.state.status = RunStatus::Voting;
15745        runner.deliberate().await.unwrap();
15746        runner.vote().await.unwrap();
15747        runner.finish_after_tally().await.unwrap();
15748        assert_eq!(runner.state.status, RunStatus::Blocked);
15749        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
15750        assert_eq!(
15751            crate::notices::run_ended(&runner.state).unwrap().since,
15752            notice.since
15753        );
15754    }
15755
15756    #[cfg(unix)]
15757    #[tokio::test]
15758    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
15759        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15760        for detail in [
15761            "fix/retained conflicts with origin/main and did not rebase: conflict",
15762            "rebased fix/retained locally but origin refused the push",
15763            "unknown failure",
15764        ] {
15765            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
15766            runner.state.record_base_sync_stop(&head);
15767            assert!(runner.state.authorize_base_sync(&task).is_err());
15768            runner.state.status = RunStatus::Voting;
15769            runner.execute_graph().await.unwrap();
15770            assert_eq!(runner.state.status, RunStatus::Blocked);
15771            assert!(!crate::handover::can_resume(&runner.state));
15772            assert_eq!(
15773                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15774                head
15775            );
15776        }
15777    }
15778
15779    #[cfg(unix)]
15780    #[tokio::test]
15781    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
15782        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15783        runner.state.authorize_base_sync(&task).unwrap();
15784        assert!(runner.state.consume_base_recovery(&head));
15785        runner.state.save().unwrap();
15786        runner.state = RunState::load(&runner.state.id).unwrap();
15787        runner.execute_graph().await.unwrap();
15788        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15789        assert_eq!(runner.state.status, RunStatus::Blocked);
15790        assert!(!runner.state.consume_base_recovery(&head));
15791        assert!(!crate::handover::can_resume(&runner.state));
15792        runner.state.authorize_base_sync(&task).unwrap();
15793        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
15794        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15795        assert_eq!(
15796            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15797            head
15798        );
15799    }
15800
15801    #[cfg(unix)]
15802    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
15803        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
15804        // This is an ordinary gate stop, not an authorized base-budget recovery.
15805        runner.state.prep_complete = Some(true);
15806        runner.state.base_sync = None;
15807        runner.state.base_sync_recovery = Default::default();
15808        runner.state.config.graph.gate_fix_rounds = 1;
15809        let mut review = review_round(true, 0, 1, 1, false, true);
15810        review.head = head.clone();
15811        review.verified_head = Some(head.clone());
15812        runner.state.reviews = vec![review];
15813        runner.state.gate = vec![CommandOutcome {
15814            command: "old gate".into(),
15815            code: Some(1),
15816            output_tail: "old failure".into(),
15817            duration_ms: 0,
15818            resource_blocked: false,
15819        }];
15820        runner.state.gate_head = Some(head.clone());
15821        runner.state.gate_ran = true;
15822        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
15823            agent: "fixture".into(),
15824            failed: runner.state.gate.clone(),
15825            notes: "spent".into(),
15826            committed: false,
15827            error: None,
15828        }];
15829        runner.state.land_approval = Some(crate::run::LandApproval {
15830            question: "old approval".into(),
15831            head: head.clone(),
15832        });
15833        runner.state.status = RunStatus::Blocked;
15834        (tmp, runner, head)
15835    }
15836
15837    #[cfg(unix)]
15838    #[tokio::test]
15839    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
15840        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
15841            let (tmp, mut runner, old) = resumed_gate_fixture().await;
15842            let phases = tmp.path().join("phases");
15843            runner.state.config.verify.e2e = vec![format!(
15844                "echo e2e >> '{}'; {}",
15845                phases.display(),
15846                if e2e_ok {
15847                    "true"
15848                } else {
15849                    "echo new-e2e-failure; false"
15850                }
15851            )];
15852            runner.state.config.verify.gate = vec![format!(
15853                "echo gate >> '{}'; {}",
15854                phases.display(),
15855                if gate_ok {
15856                    "true"
15857                } else {
15858                    "echo new-gate-failure; false"
15859                }
15860            )];
15861            runner.state.save().unwrap();
15862            runner.state = RunState::load(&runner.state.id).unwrap();
15863            runner.sync_to_base().await.unwrap();
15864            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15865            assert_ne!(head, old);
15866            // Reentry after Git updated but before verification was invalidated.
15867            runner.state = RunState::load(&runner.state.id).unwrap();
15868            runner.execute_graph().await.unwrap();
15869            assert_eq!(runner.state.gate_fixes.len(), 1);
15870            assert_eq!(runner.state.reviews.len(), 1);
15871            assert_eq!(runner.state.reviews[0].head, old);
15872            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
15873            assert_eq!(runner.state.verification_history.len(), 1);
15874            let history = &runner.state.verification_history[0];
15875            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
15876            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
15877            assert!(history.archived_at.is_some());
15878            assert_eq!(history.fixes_spent, Some(1));
15879            assert_eq!(history.gate[0].output_tail, "old failure");
15880            let archive_events: Vec<_> = runner
15881                .state
15882                .events
15883                .iter()
15884                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
15885                .collect();
15886            assert_eq!(archive_events.len(), 1);
15887            assert!(archive_events[0].message.contains(&short(&old)));
15888            assert!(archive_events[0].message.contains(&short(&head)));
15889            assert!(
15890                archive_events[0]
15891                    .message
15892                    .contains("1 gate-fix round(s) already spent stay spent")
15893            );
15894            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
15895            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
15896            if e2e_ok {
15897                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
15898                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
15899            } else {
15900                assert!(!runner.state.gate_ran);
15901            }
15902            assert_eq!(
15903                runner.state.status,
15904                if e2e_ok && gate_ok {
15905                    RunStatus::Ready
15906                } else {
15907                    RunStatus::Blocked
15908                }
15909            );
15910            if !e2e_ok || !gate_ok {
15911                let cause = runner.state.verification_blocking_cause().unwrap();
15912                assert!(cause.contains(&head));
15913                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
15914                // A later successful progress event must not become the cause.
15915                runner.state.event("land", "rebased successfully");
15916                let notice = crate::notices::run_ended(&runner.state).unwrap();
15917                assert!(!format!("{notice:?}").contains("rebased successfully"));
15918                assert!(runner.state.merge.is_none());
15919            }
15920            let before = std::fs::read_to_string(&phases).unwrap();
15921            runner.state.save().unwrap();
15922            runner.state = RunState::load(&runner.state.id).unwrap();
15923            runner.review_loop().await.unwrap();
15924            runner.gate().await.unwrap();
15925            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
15926            assert_eq!(runner.state.verification_history.len(), 1);
15927            let history = &runner.state.verification_history[0];
15928            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
15929            assert!(history.archived_at.is_some());
15930            assert_eq!(history.fixes_spent, Some(1));
15931        }
15932    }
15933
15934    #[cfg(unix)]
15935    #[tokio::test]
15936    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
15937        let (tmp, mut runner, head) = resumed_gate_fixture().await;
15938        let marker = tmp.path().join("gate-ran");
15939        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
15940        runner.review_loop().await.unwrap();
15941        runner.gate().await.unwrap();
15942        assert_eq!(runner.state.status, RunStatus::Blocked);
15943        assert!(!marker.exists());
15944        assert!(runner.state.verification_history.is_empty());
15945        runner.state.gate_head = None;
15946        runner.gate().await.unwrap();
15947        assert!(marker.exists());
15948        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
15949        assert_eq!(runner.state.verification_history.len(), 1);
15950        assert_eq!(runner.state.gate_fixes.len(), 1);
15951    }
15952
15953    #[cfg(unix)]
15954    #[tokio::test]
15955    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
15956        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
15957        runner.sync_to_base().await.unwrap();
15958        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15959        runner.state.reconcile_verification_head(&head);
15960        // Both crash-before-command and resource wait are unresolved evidence.
15961        for outcomes in [
15962            Vec::new(),
15963            vec![CommandOutcome {
15964                command: "true".into(),
15965                code: None,
15966                output_tail: "cache unavailable".into(),
15967                duration_ms: 0,
15968                resource_blocked: true,
15969            }],
15970        ] {
15971            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
15972                head: head.clone(),
15973                passed: false,
15974                outcomes,
15975            });
15976            runner.state.gate_ran = false;
15977            runner.state.gate.clear();
15978            runner.state.save().unwrap();
15979            runner.state = RunState::load(&runner.state.id).unwrap();
15980            runner.merge().await.unwrap();
15981            assert!(runner.state.merge.is_none());
15982            runner.review_loop().await.unwrap();
15983            runner.gate().await.unwrap();
15984            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
15985            assert!(runner.state.gate_allows_progress());
15986            assert_eq!(runner.state.gate_fixes.len(), 1);
15987            assert!(runner.state.merge.is_none());
15988        }
15989        // Moving the winner after a successful gate cannot publish an unchecked SHA.
15990        git::git(
15991            &runner.state.repo,
15992            &["commit", "--allow-empty", "-m", "external update"],
15993        )
15994        .await
15995        .unwrap();
15996        runner.merge().await.unwrap();
15997        assert!(runner.state.merge.is_none());
15998        assert_eq!(runner.state.status, RunStatus::Blocked);
15999    }
16000
16001    #[cfg(unix)]
16002    #[tokio::test]
16003    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
16004        for waiting_on_gate in [false, true] {
16005            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
16006            runner.sync_to_base().await.unwrap();
16007            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16008            runner.state.reconcile_verification_head(&head);
16009            if waiting_on_gate {
16010                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16011                    head: head.clone(),
16012                    passed: true,
16013                    outcomes: Vec::new(),
16014                });
16015            }
16016            let cache = tmp.path().join("cache");
16017            runner.state.config.verify.e2e =
16018                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
16019            runner.state.config.verify.gate = vec!["true".into()];
16020            runner.state.config.graph.timeout_resource_wait = 1;
16021            let owner =
16022                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
16023            let guard =
16024                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
16025                    crate::cache::AcquireOutcome::Acquired(g) => g,
16026                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
16027                };
16028            runner.review_loop().await.unwrap();
16029            runner.gate().await.unwrap();
16030            assert!(!runner.state.gate_ran);
16031            assert_eq!(runner.state.gate_fixes.len(), 1);
16032            assert!(runner.state.resource_block.is_some());
16033            runner.merge().await.unwrap();
16034            assert!(runner.state.merge.is_none());
16035            drop(guard);
16036            runner.state = RunState::load(&runner.state.id).unwrap();
16037            // The driver clears the previous pass's resource stop on resume.
16038            runner.state.resource_block = None;
16039            runner.review_loop().await.unwrap();
16040            runner.gate().await.unwrap();
16041            assert!(runner.state.gate_allows_progress());
16042            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16043            assert_eq!(runner.state.gate_fixes.len(), 1);
16044        }
16045    }
16046
16047    #[cfg(unix)]
16048    #[tokio::test]
16049    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
16050        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16051        runner.sync_to_base().await.unwrap();
16052        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16053        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
16054        // Verification passes before pre_gate and fails on the mechanically changed head.
16055        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
16056        runner.review_loop().await.unwrap();
16057        runner.gate().await.unwrap();
16058        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16059        assert_ne!(before, after);
16060        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
16061        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
16062        assert!(!runner.state.gate_allows_progress());
16063        runner.merge().await.unwrap();
16064        assert!(runner.state.merge.is_none());
16065        assert!(
16066            runner
16067                .state
16068                .verification_history
16069                .iter()
16070                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
16071        );
16072
16073        // An incomplete review is still an independent hold on a changed head.
16074        runner.state.reviews[0].clean = false;
16075        runner.state.reviews[0].answered = 0;
16076        runner.state.config.verify.e2e = vec!["true".into()];
16077        runner.review_loop().await.unwrap();
16078        runner.gate().await.unwrap();
16079        runner.merge().await.unwrap();
16080        assert_eq!(runner.state.status, RunStatus::Blocked);
16081        assert!(runner.state.merge.is_none());
16082        assert!(
16083            runner
16084                .state
16085                .verification_blocking_cause()
16086                .unwrap()
16087                .contains("review blocked")
16088        );
16089    }
16090
16091    #[cfg(unix)]
16092    #[tokio::test]
16093    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
16094        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
16095        runner.state.gate_head = None;
16096        runner.state.config.verify.gate =
16097            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
16098        runner.review_loop().await.unwrap();
16099        runner.gate().await.unwrap();
16100        assert_ne!(
16101            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16102            old
16103        );
16104        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
16105        assert!(!runner.state.gate_allows_progress());
16106        assert_eq!(runner.state.status, RunStatus::Blocked);
16107        runner.merge().await.unwrap();
16108        assert!(runner.state.merge.is_none());
16109    }
16110
16111    #[cfg(unix)]
16112    #[tokio::test]
16113    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
16114        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16115        let mut old_review = review_round(true, 0, 1, 1, false, true);
16116        old_review.head = head.clone();
16117        old_review.verified_head = Some(head.clone());
16118        runner.state.reviews = vec![old_review];
16119        runner.state.gate = vec![CommandOutcome {
16120            command: "old gate".into(),
16121            code: None,
16122            output_tail: "timed out after 2400s".into(),
16123            duration_ms: 2_400_000,
16124            resource_blocked: false,
16125        }];
16126        runner.state.gate_ran = true;
16127        runner.state.gate_head = Some(head.clone());
16128        runner.state.land_approval = Some(crate::run::LandApproval {
16129            question: "old-head-approval".into(),
16130            head: head.clone(),
16131        });
16132        runner.state.authorize_base_sync(&task).unwrap();
16133        runner.sync_to_base().await.unwrap();
16134        assert!(!runner.state.base_sync_blocked());
16135        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16136        assert_ne!(head, fresh);
16137        assert!(runner.state.reviews.is_empty());
16138        assert!(!runner.state.gate_ran);
16139        let saved = &runner.state.base_sync_recovery.authorizations[0];
16140        assert_eq!(saved.reviews[0].head, head);
16141        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
16142        assert!(saved.gate_ran);
16143        assert!(!saved.gate[0].ok());
16144        runner.review_loop().await.unwrap();
16145        runner.gate().await.unwrap();
16146        assert_eq!(runner.state.reviews[0].head, fresh);
16147        assert_eq!(
16148            runner.state.reviews[0].verified_head.as_deref(),
16149            Some(fresh.as_str())
16150        );
16151        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
16152        assert!(runner.state.gate_ran);
16153        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16154        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
16155        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
16156        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16157        assert_eq!(
16158            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
16159            "retained change"
16160        );
16161    }
16162
16163    #[cfg(unix)]
16164    #[tokio::test]
16165    async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
16166        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16167        let queue = crate::queue::Queue::open();
16168        let mut owner = queue.get(&task).unwrap();
16169        owner.release_operator("test-resume", None);
16170        queue.put(&mut owner).unwrap();
16171        let generation = owner.resumption.as_ref().unwrap().generation;
16172        runner.state.task_instruction = Some(owner.instruction.clone());
16173        runner.state.driver_exited = true;
16174        runner.state.save().unwrap();
16175        let home = crate::run::home();
16176        let _claim = queue.claim(&task).unwrap();
16177        owner.resumption.as_mut().unwrap().at =
16178            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
16179        queue.put(&mut owner).unwrap();
16180        assert!(
16181            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16182                .await
16183                .is_err(),
16184            "an approval from before the stop grants nothing"
16185        );
16186        owner.resumption.as_mut().unwrap().at = Timestamp::now();
16187        queue.put(&mut owner).unwrap();
16188        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16189            .await
16190            .unwrap();
16191        assert!(
16192            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16193                .await
16194                .is_err()
16195        );
16196        runner.state = RunState::load(&runner.state.id).unwrap();
16197        assert_eq!(
16198            runner.state.base_sync_recovery.authorizations[0].generation,
16199            Some(generation)
16200        );
16201        runner.sync_to_base().await.unwrap();
16202        let repo = runner.state.repo.clone();
16203        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16204        git::git(&repo, &["checkout", "main"]).await.unwrap();
16205        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
16206        git::git(&repo, &["add", "."]).await.unwrap();
16207        git::git(&repo, &["commit", "-m", "move base again"])
16208            .await
16209            .unwrap();
16210        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16211        git::git(&repo, &["checkout", "fix/retained"])
16212            .await
16213            .unwrap();
16214        runner.finish_after_tally().await.unwrap();
16215        assert_eq!(runner.state.status, RunStatus::Blocked);
16216        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16217        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16218        assert!(!runner.state.base_recovery_pending());
16219        assert!(!crate::handover::can_resume(&runner.state));
16220        assert!(!runner.state.gate_ran);
16221        // Even replaying the generation with a later timestamp grants nothing.
16222        owner.resumption.as_mut().unwrap().at = Timestamp::now();
16223        queue.put(&mut owner).unwrap();
16224        assert!(
16225            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16226                .await
16227                .is_err()
16228        );
16229        runner.state = RunState::load(&runner.state.id).unwrap();
16230        runner.sync_to_base().await.unwrap();
16231        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16232        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
16233        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16234    }
16235
16236    #[cfg(unix)]
16237    #[tokio::test]
16238    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
16239        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
16240        runner.state.base_sync = None;
16241        runner.state.status = RunStatus::Blocked;
16242        let mut second = runner.state.candidates[0].clone();
16243        second.label = 'B';
16244        runner.state.candidates.push(second);
16245        assert!(runner.state.votes.is_empty());
16246        assert!(runner.state.deliberation.is_empty());
16247        runner.deliberate().await.unwrap();
16248        runner.vote().await.unwrap();
16249        assert_eq!(runner.state.status, RunStatus::Blocked);
16250        assert!(runner.state.votes.is_empty());
16251        assert!(runner.state.deliberation.is_empty());
16252    }
16253
16254    /// A minimal, git-free `Runner` at a given status — `settle_questions`
16255    /// reads nothing else off it.
16256    fn runner_at(status: RunStatus) -> Runner {
16257        let mut state = RunState::new(
16258            PathBuf::from("/nonexistent/repo"),
16259            "main".to_owned(),
16260            "deadbeef".to_owned(),
16261            "task".to_owned(),
16262            Config::default(),
16263        );
16264        // These fixtures share one home and save concurrently. Short clock
16265        // IDs can collide, making an ownership test read another test's run.
16266        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
16267        state.id = format!(
16268            "20000101-000000-{:04x}",
16269            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
16270        );
16271        state.status = status;
16272        Runner {
16273            state,
16274            roles: ResolvedRoles {
16275                implementers: Vec::new(),
16276                judges: Vec::new(),
16277                reviewers: Vec::new(),
16278                fixer: None,
16279                conductor: conductor(),
16280                implementer_roster: Vec::new(),
16281                judge_roster: Vec::new(),
16282                reviewer_roster: Vec::new(),
16283            },
16284            sem: Arc::new(Semaphore::new(1)),
16285            pause: Pause::new(),
16286            interrupt: Pause::new(),
16287        }
16288    }
16289
16290    #[tokio::test]
16291    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
16292        crate::run::pin_test_home();
16293        let mut runner = runner_at(RunStatus::Gating);
16294        runner.state.save().unwrap();
16295        let id = runner.state.id.clone();
16296        let passes = runner.state.passes.len();
16297        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16298        let mut claiming = Box::pin(runner.claim_driver());
16299        std::future::poll_fn(|cx| {
16300            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
16301            std::task::Poll::Ready(())
16302        })
16303        .await;
16304        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
16305        drop(owner);
16306        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
16307            .await
16308            .unwrap()
16309            .unwrap();
16310        assert_eq!(execution.runner.state.passes.len(), passes + 1);
16311        assert!(execution.runner.state.resource_block.is_none());
16312        assert_eq!(
16313            execution.runner.state.liveness(false),
16314            crate::run::Liveness::Live
16315        );
16316        drop(execution);
16317        assert!(RunState::load(&id).unwrap().driver_exited);
16318    }
16319
16320    #[tokio::test]
16321    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
16322        crate::run::pin_test_home();
16323        let mut runner = runner_at(RunStatus::Gating);
16324        runner.state.save().unwrap();
16325        // run.json can commit, but the subsequent lease metadata write fails.
16326        // Cleanup must run after releasing the short ownership lock.
16327        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
16328        assert!(runner.claim_driver().await.is_err());
16329        assert!(runner.state.driver_exited);
16330        assert_eq!(runner.state.driver_pid, None);
16331        let disk = RunState::load(&runner.state.id).unwrap();
16332        assert!(disk.driver_exited);
16333        assert_eq!(disk.driver_pid, None);
16334        assert_eq!(disk.status, RunStatus::Gating);
16335    }
16336
16337    #[tokio::test]
16338    async fn an_early_graph_error_releases_its_driver() {
16339        crate::run::pin_test_home();
16340        let mut runner = runner_at(RunStatus::Blocked);
16341        runner.state.config.graph.selected_implementer = Some("selected".into());
16342        runner.state.save().unwrap();
16343        assert!(runner.execute().await.is_err());
16344        let disk = RunState::load(&runner.state.id).unwrap();
16345        assert!(disk.driver_exited);
16346        assert_eq!(disk.driver_pid, None);
16347        assert_eq!(disk.status, RunStatus::Blocked);
16348    }
16349
16350    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
16351        crate::run::MergeRepair {
16352            question: "repair-question".into(),
16353            head: "old-head".into(),
16354            pr: "https://github.com/o/r/pull/42".into(),
16355            reviews: vec![review_round(false, 1, 1, 1, false, true)],
16356            gate: vec![outcome(Some(0), false)],
16357            gate_head: Some("old-head".into()),
16358            gate_fixes: Vec::new(),
16359            pre_gate: Vec::new(),
16360            phase,
16361            failure: None,
16362        }
16363    }
16364
16365    #[tokio::test]
16366    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
16367        crate::run::pin_test_home();
16368        let mut runner = runner_at(RunStatus::Landing);
16369        runner.state.merge_repairs.push(merge_repair_request(
16370            crate::run::MergeRepairPhase::Requested,
16371        ));
16372        runner.state.save().unwrap();
16373        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
16374        assert!(
16375            runner
16376                .execute()
16377                .await
16378                .unwrap_err()
16379                .to_string()
16380                .contains("already running")
16381        );
16382        assert_eq!(
16383            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16384            crate::run::MergeRepairPhase::Requested
16385        );
16386        drop(claim);
16387        let mut disk = runner.state.clone();
16388        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
16389        disk.save().unwrap();
16390        assert!(
16391            runner
16392                .execute()
16393                .await
16394                .unwrap_err()
16395                .to_string()
16396                .contains("decision changed")
16397        );
16398        assert_eq!(
16399            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16400            crate::run::MergeRepairPhase::Complete
16401        );
16402    }
16403
16404    #[test]
16405    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
16406        crate::run::pin_test_home();
16407        let mut runner = runner_at(RunStatus::Landing);
16408        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
16409        runner.state.reviews = request.reviews.clone();
16410        runner.state.gate = request.gate.clone();
16411        runner.state.gate_ran = true;
16412        runner.state.land_approval = Some(crate::run::LandApproval {
16413            question: "old".into(),
16414            head: "old-head".into(),
16415        });
16416        runner.state.merge_repairs.push(request);
16417        runner.begin_merge_repair_review(0).unwrap();
16418        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
16419        assert!(!runner.state.gate_status().ok());
16420        assert!(runner.state.land_approval.is_none());
16421        let disk = RunState::load(&runner.state.id).unwrap();
16422        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
16423        assert!(disk.merge_repairs[0].gate[0].ok());
16424        assert_eq!(
16425            disk.merge_repairs[0].phase,
16426            crate::run::MergeRepairPhase::Reviewing
16427        );
16428        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
16429        assert!(disk.reviews.is_empty());
16430    }
16431
16432    #[tokio::test]
16433    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
16434        crate::run::pin_test_home();
16435        for phase in [
16436            crate::run::MergeRepairPhase::Fixing,
16437            crate::run::MergeRepairPhase::Requested,
16438        ] {
16439            let mut runner = runner_at(RunStatus::Landing);
16440            runner.state.config.graph.land_rounds = 0;
16441            runner.state.merge_repairs.push(merge_repair_request(phase));
16442            runner.state.save().unwrap();
16443            assert!(!runner.repair_merge_review().await.unwrap());
16444            assert_eq!(runner.state.status, RunStatus::Blocked);
16445            assert_eq!(
16446                runner.state.merge_repairs[0].phase,
16447                crate::run::MergeRepairPhase::Failed
16448            );
16449            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
16450            assert!(
16451                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
16452                    "interrupted"
16453                } else {
16454                    "budget exhausted"
16455                })
16456            );
16457            assert!(runner.state.merge.is_none());
16458        }
16459    }
16460
16461    /// `park_here` folding in the reason `Pause::park_because` recorded -
16462    /// this is what lets an operator reading a run's events tell an
16463    /// interrupt-driven park from an ordinary shutdown park.
16464    #[test]
16465    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
16466        crate::run::pin_test_home();
16467        let mut runner = runner_at(RunStatus::Implementing);
16468        let interrupt = Pause::new();
16469        runner.watch_interrupt(interrupt.clone());
16470
16471        interrupt.park_because("task a1b2 asked to run first");
16472
16473        assert!(runner.park_here().expect("park_here"));
16474        assert!(runner.state.parked);
16475        let last = runner.state.events.last().expect("a park event");
16476        assert_eq!(last.node, "park");
16477        assert!(
16478            last.message.contains("task a1b2 asked to run first"),
16479            "expected the interrupt reason in {:?}",
16480            last.message
16481        );
16482    }
16483
16484    /// The cause is stored on the pass when it parks, and the first cause a
16485    /// shared pause was given is the one that stays.
16486    #[test]
16487    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
16488        crate::run::pin_test_home();
16489        let mut runner = runner_at(RunStatus::Implementing);
16490        runner.state.begin_pass();
16491        let shutdown = Pause::new();
16492        runner.on_pause(shutdown.clone());
16493        let upgrade = ParkCause::Upgrade {
16494            from: "0.121.0".to_owned(),
16495            to: Some("0.121.1".to_owned()),
16496        };
16497        shutdown.park_for(upgrade.clone());
16498        shutdown.park_for(ParkCause::Operator);
16499
16500        assert!(runner.park_here().expect("park_here"));
16501        assert_eq!(
16502            runner.state.passes[0].end,
16503            Some(crate::run::PassEnd::Parked { cause: upgrade })
16504        );
16505        let last = runner.state.events.last().expect("a park event");
16506        assert!(last.message.contains("0.121.0"), "{}", last.message);
16507    }
16508
16509    #[tokio::test]
16510    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
16511        crate::run::pin_test_home();
16512        let mut runner = runner_at(RunStatus::Reviewing);
16513        runner.state.save().unwrap();
16514        let mut execution = runner.claim_driver().await.expect("first execution");
16515        assert_eq!(execution.runner.state.passes.len(), 1);
16516        execution
16517            .driver
16518            .finish(&mut execution.runner.state)
16519            .await
16520            .unwrap();
16521    }
16522
16523    #[tokio::test]
16524    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
16525        crate::run::pin_test_home();
16526        let mut runner = runner_at(RunStatus::Reviewing);
16527        runner.state.config.agents = vec![conductor()];
16528        runner.state.config.graph.timeout_resource_wait = 0;
16529        runner.state.save().unwrap();
16530        let id = runner.state.id.clone();
16531        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16532        assert!(runner.claim_driver().await.is_err());
16533        drop(owner);
16534        drop(runner);
16535        let mut resumed = Runner::resume(&id).unwrap();
16536        assert_eq!(resumed.state.driver_claimed, Some(false));
16537        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
16538        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
16539        let mut execution = resumed
16540            .claim_driver()
16541            .await
16542            .expect("retry initial claim after reload");
16543        assert_eq!(execution.runner.state.driver_claimed, Some(true));
16544        assert_eq!(execution.runner.state.passes.len(), 1);
16545        execution
16546            .driver
16547            .finish(&mut execution.runner.state)
16548            .await
16549            .unwrap();
16550    }
16551
16552    #[tokio::test]
16553    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
16554        crate::run::pin_test_home();
16555        let mut runner = runner_at(RunStatus::Reviewing);
16556        runner.state.config.agents = vec![conductor()];
16557        runner.state.save().unwrap();
16558        let id = runner.state.id.clone();
16559        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
16560        let mut resumed = Runner::resume(&id).unwrap();
16561        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
16562        assert!(resumed.claim_driver().await.is_err());
16563        assert!(RunState::load(&id).unwrap().passes.is_empty());
16564        drop(lease);
16565        let mut execution = resumed.claim_driver().await.unwrap();
16566        execution
16567            .driver
16568            .finish(&mut execution.runner.state)
16569            .await
16570            .unwrap();
16571    }
16572
16573    #[tokio::test]
16574    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
16575        crate::run::pin_test_home();
16576        let mut runner = runner_at(RunStatus::Reviewing);
16577        let state = &mut runner.state;
16578        state.config.agents = vec![conductor()];
16579        state.driver_claimed = None;
16580        state.driver_pid = Some(std::process::id());
16581        state.driver_started_at = None;
16582        state.begin_pass();
16583        state.save().unwrap();
16584        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
16585        std::fs::write(
16586            state.dir().join("driver.json"),
16587            serde_json::to_vec(&serde_json::json!({
16588                "pid": state.driver_pid, "started_at": null,
16589                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
16590            }))
16591            .unwrap(),
16592        )
16593        .unwrap();
16594        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
16595        drop(lease);
16596        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
16597        let mut resumed = Runner::resume(&state.id).unwrap();
16598        assert!(!resumed.state.driver_exited);
16599        let mut execution = resumed
16600            .claim_driver()
16601            .await
16602            .expect("lease proves previous execution ended");
16603        assert_eq!(execution.runner.state.passes.len(), 2);
16604        execution
16605            .driver
16606            .finish(&mut execution.runner.state)
16607            .await
16608            .unwrap();
16609    }
16610
16611    #[tokio::test]
16612    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
16613        crate::run::pin_test_home();
16614        let mut runner = runner_at(RunStatus::Reviewing);
16615        let mut json = serde_json::to_value(&runner.state).unwrap();
16616        json.as_object_mut().unwrap().remove("driver_claimed");
16617        runner.state = serde_json::from_value(json).unwrap();
16618        assert_eq!(runner.state.driver_claimed, None);
16619        runner.state.save().unwrap();
16620        let mut execution = runner
16621            .claim_driver()
16622            .await
16623            .expect("the execution lease, not a missing marker, excludes a second driver");
16624        assert_eq!(execution.runner.state.passes.len(), 1);
16625        execution
16626            .driver
16627            .finish(&mut execution.runner.state)
16628            .await
16629            .unwrap();
16630    }
16631
16632    #[tokio::test]
16633    async fn driver_admission_refuses_a_proven_live_driver() {
16634        crate::run::pin_test_home();
16635        let mut runner = runner_at(RunStatus::Reviewing);
16636        let pid = std::process::id();
16637        runner.state.driver_pid = Some(pid);
16638        runner.state.driver_started_at = crate::proc::process_started_at(pid);
16639        assert!(runner.state.driver_started_at.is_some());
16640        runner.state.save().unwrap();
16641        let error = runner
16642            .claim_driver()
16643            .await
16644            .err()
16645            .expect("a live driver must refuse");
16646        assert!(error.to_string().contains("live driver"), "{error:#}");
16647        let saved = RunState::load(&runner.state.id).unwrap();
16648        assert_eq!(saved.driver_pid, Some(pid));
16649        assert_eq!(saved.status, RunStatus::Reviewing);
16650        assert!(saved.passes.is_empty());
16651    }
16652
16653    #[test]
16654    fn parking_resource_halt_persists_the_pass_once() {
16655        crate::run::pin_test_home();
16656        let mut runner = runner_at(RunStatus::Reviewing);
16657        runner.state.begin_pass();
16658        runner.state.resource_block = Some(crate::run::ResourceBlock {
16659            node: "fix".into(),
16660            kind: "shared build cache".into(),
16661            detail: "cancelled".into(),
16662            at: Timestamp::now(),
16663            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
16664        });
16665        let cause = ParkCause::Upgrade {
16666            from: "old".into(),
16667            to: Some("new".into()),
16668        };
16669        runner.pause.park_for(cause.clone());
16670        // Both direct fixer exits and graph resource boundaries must complete
16671        // parking, rather than returning a stale Reviewing record.
16672        runner.stop_for_resource().unwrap();
16673        assert!(runner.resource_halt().unwrap());
16674        assert!(runner.park_here().unwrap());
16675        let saved = RunState::load(&runner.state.id).unwrap();
16676        assert!(saved.parked);
16677        assert_eq!(saved.status, RunStatus::Reviewing);
16678        assert!(saved.passes[0].ended_at.is_some());
16679        assert_eq!(
16680            saved.passes[0].end,
16681            Some(crate::run::PassEnd::Parked {
16682                cause: cause.clone()
16683            })
16684        );
16685        assert_eq!(
16686            saved.resource_block.unwrap().cause,
16687            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
16688        );
16689        assert_eq!(
16690            saved
16691                .events
16692                .iter()
16693                .filter(|event| event.node == "park")
16694                .count(),
16695            1
16696        );
16697    }
16698
16699    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
16700    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
16701    /// cleared) must not make a *different* run - one only watching its own,
16702    /// unshared interrupt `Pause` - see itself as parked. If a future change
16703    /// ever collapsed these back into one handle, the interrupt scheduler
16704    /// would park every run for the rest of the daemon's life, not just the
16705    /// one it meant to interrupt.
16706    #[test]
16707    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
16708        crate::run::pin_test_home();
16709        let mut runner = runner_at(RunStatus::Implementing);
16710        let shutdown = Pause::new();
16711        runner.on_pause(shutdown.clone());
16712        let interrupt = Pause::new();
16713        runner.watch_interrupt(interrupt.clone());
16714
16715        // Nobody has asked for anything yet.
16716        assert!(!runner.park_here().expect("park_here"));
16717        assert!(!runner.state.parked);
16718
16719        // Only the interrupt handle fires; the shutdown handle stays clear.
16720        interrupt.park_because("test");
16721        assert!(!shutdown.parked());
16722        assert!(runner.park_here().expect("park_here"));
16723    }
16724
16725    /// The property every prior attempt at this feature failed to pin down:
16726    /// asking a run to park while one of its nodes has a real, in-flight
16727    /// async operation running (an agent call, in production) must not cut
16728    /// that operation short. `park_here` is only ever consulted *between*
16729    /// `execute`'s node calls - see its own doc - so nothing inside a node
16730    /// can observe a park request until the node itself returns. This proves
16731    /// that structurally, with real `tokio` concurrency and a channel
16732    /// handshake (never a sleep, which would only prove "usually", not
16733    /// "cannot"): the "node" below reports that it has genuinely started,
16734    /// and only then is the park requested; the node still has to be told to
16735    /// finish before `park_here` is ever called, exactly mirroring every
16736    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
16737    /// in `execute`.
16738    #[tokio::test]
16739    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
16740        crate::run::pin_test_home();
16741        let mut runner = runner_at(RunStatus::Implementing);
16742        let interrupt = Pause::new();
16743        runner.watch_interrupt(interrupt.clone());
16744
16745        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
16746        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
16747
16748        // Stands in for one node's in-flight agent call: it proves it has
16749        // genuinely started, then blocks - exactly as a spawned CLI process
16750        // does - until told to finish.
16751        let node = async move {
16752            started_tx.send(()).expect("send started");
16753            finish_rx.await.expect("recv finish");
16754            "node finished"
16755        };
16756
16757        let interrupter = async move {
16758            started_rx.await.expect("recv started");
16759            // The call is now genuinely in flight. Ask it to park.
16760            interrupt.park_because("higher-priority task waiting");
16761            // Nothing the node does can observe this yet - there is no
16762            // check inside it, by construction - so let the executor run
16763            // anything pending and then let the node finish on its own.
16764            tokio::task::yield_now().await;
16765            finish_tx.send(()).expect("send finish");
16766        };
16767
16768        let (node_result, ()) = tokio::join!(node, interrupter);
16769        assert_eq!(
16770            node_result, "node finished",
16771            "the in-flight call ran to completion"
16772        );
16773
16774        // Only now, at the boundary the real `execute` would check right
16775        // after this node, does the park take effect.
16776        assert!(runner.park_here().expect("park_here"));
16777        assert!(runner.state.parked);
16778    }
16779
16780    /// A run parked mid-competition carries every field it had accumulated
16781    /// through the exact same disk round-trip an ordinary resume uses -
16782    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
16783    /// Nothing about parking for an interrupt is a special case of that path;
16784    /// this is what proves it rather than assuming it.
16785    #[test]
16786    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
16787        crate::run::pin_test_home();
16788        let mut runner = runner_at(RunStatus::Judging);
16789        // `Runner::resume` re-resolves roles from the saved config, which
16790        // refuses an empty roster - give it the same minimal one `conductor`
16791        // itself uses.
16792        runner.state.config.agents = vec![conductor()];
16793        runner.state.candidates = vec![Candidate {
16794            index: 0,
16795            label: 'A',
16796            agent: "alpha".to_owned(),
16797            branch: "magi/x/A".to_owned(),
16798            worktree: PathBuf::from("/nonexistent/worktree"),
16799            summary: "did the thing".to_owned(),
16800            stat: "1 file changed".to_owned(),
16801            files: 1,
16802            commits: 1,
16803            empty: false,
16804            failed: None,
16805            verified_noop: None,
16806            duration_ms: 1234,
16807            folded: false,
16808        }];
16809        let run_id = runner.state.id.clone();
16810
16811        let interrupt = Pause::new();
16812        runner.watch_interrupt(interrupt.clone());
16813        interrupt.park_because("task c3d4 asked to run first");
16814        assert!(runner.park_here().expect("park_here"));
16815
16816        let resumed = Runner::resume(&run_id).expect("resume");
16817        assert_eq!(resumed.state.candidates.len(), 1);
16818        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
16819        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
16820        assert_eq!(resumed.state.status, runner.state.status);
16821        assert!(
16822            resumed.state.parked,
16823            "still parked until `execute` actually walks the graph again"
16824        );
16825        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
16826    }
16827
16828    /// A fresh open question on `run`, stored and handed back for assertions.
16829    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
16830        let mut q = ask::Question::new(
16831            run.to_owned(),
16832            "implement".to_owned(),
16833            "impl-A".to_owned(),
16834            "Which storage backend should the cache use?".to_owned(),
16835            String::new(),
16836            vec!["SQLite".to_owned(), "Redis".to_owned()],
16837        );
16838        store.put(&mut q).unwrap();
16839        q
16840    }
16841
16842    #[test]
16843    fn a_failed_runs_open_question_is_abandoned() {
16844        ask_test_home();
16845        let store = ask::Questions::open();
16846        let mut runner = runner_at(RunStatus::Failed);
16847        let run = runner.state.id.clone();
16848        let q = ask_open_question(&store, &run);
16849
16850        runner.settle_questions();
16851
16852        let back = store.get(&q.id).unwrap();
16853        assert!(
16854            !back.status.open(),
16855            "the seat that asked died with the run; nobody is left to read an answer"
16856        );
16857        assert!(
16858            back.detail.contains(&run) && back.detail.contains("failed"),
16859            "the reason names what the run became, not just that it is gone: {}",
16860            back.detail
16861        );
16862    }
16863
16864    #[test]
16865    fn a_merged_runs_open_question_is_abandoned_too() {
16866        ask_test_home();
16867        let store = ask::Questions::open();
16868        // A run that finishes cleanly still leaves nobody to read an answer -
16869        // this is not only a failure-path cleanup.
16870        for status in [RunStatus::Merged, RunStatus::Ready] {
16871            let mut runner = runner_at(status);
16872            let run = runner.state.id.clone();
16873            let q = ask_open_question(&store, &run);
16874
16875            runner.settle_questions();
16876
16877            let back = store.get(&q.id).unwrap();
16878            assert!(
16879                !back.status.open(),
16880                "{status:?} run's question must not outlive the run"
16881            );
16882        }
16883    }
16884
16885    #[test]
16886    fn a_still_resumable_runs_open_question_is_left_alone() {
16887        ask_test_home();
16888        let store = ask::Questions::open();
16889        // `Blocked` and `Stalled` can still be resumed — the candidates, the
16890        // review round and the seat sessions are all still on disk — so a
16891        // question asked mid-round may yet get a real answer from a real
16892        // resume. Sweeping it here would be exactly the failure mode this
16893        // whole feature exists to avoid on the other side.
16894        for status in [RunStatus::Blocked, RunStatus::Stalled] {
16895            let mut runner = runner_at(status);
16896            let run = runner.state.id.clone();
16897            let q = ask_open_question(&store, &run);
16898
16899            runner.settle_questions();
16900
16901            let back = store.get(&q.id).unwrap();
16902            assert!(
16903                back.status.open(),
16904                "{status:?} is still alive; the question must still be waiting"
16905            );
16906        }
16907    }
16908
16909    #[test]
16910    fn settle_questions_never_touches_an_already_answered_question() {
16911        ask_test_home();
16912        let store = ask::Questions::open();
16913        let mut runner = runner_at(RunStatus::Failed);
16914        let run = runner.state.id.clone();
16915        let mut q = ask_open_question(&store, &run);
16916        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
16917            .unwrap();
16918        store.put(&mut q).unwrap();
16919
16920        // Called twice, the way a crash-recovered daemon reclaim and the
16921        // graph's own cleanup both can for the same run — `abandon_for_run`
16922        // only ever touches what is still open, so this must be inert both
16923        // times, not merely the second.
16924        runner.settle_questions();
16925        runner.settle_questions();
16926
16927        let back = store.get(&q.id).unwrap();
16928        assert_eq!(
16929            back.status,
16930            ask::QuestionStatus::Answered,
16931            "a real answer is a decision on record, never overwritten by a sweep"
16932        );
16933    }
16934
16935    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
16936    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
16937    /// without merging, whose winner is still the operator's answer to read.
16938    /// Nothing previously called `fold_run` itself with a real `tally`, so
16939    /// this is the first test to pin down the one distinction the whole
16940    /// automatic-fold feature depends on: the winner's worktree and branch
16941    /// must survive, everything else sharing the run's worktree bay - a
16942    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
16943    /// second sweep treats every non-winner directory under the bay alike -
16944    /// must not.
16945    #[tokio::test]
16946    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
16947        crate::run::pin_test_home();
16948        let tmp = tempfile::tempdir().expect("tempdir");
16949        let repo = tmp.path().join("repo");
16950        std::fs::create_dir_all(&repo).unwrap();
16951        init_repo(&repo);
16952
16953        let mut config = Config::default();
16954        config.graph.worktree_root = Some(tmp.path().join("wt"));
16955
16956        let mut state = RunState::new(
16957            repo.clone(),
16958            "main".to_owned(),
16959            "deadbeef".to_owned(),
16960            "task".to_owned(),
16961            config,
16962        );
16963        let root = state.worktree_root();
16964        let wt_a = root.join("cand-A");
16965        let wt_b = root.join("cand-B");
16966        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
16967            .await
16968            .expect("worktree A");
16969        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
16970            .await
16971            .expect("worktree B");
16972
16973        state.candidates = vec![
16974            Candidate {
16975                index: 0,
16976                label: 'A',
16977                agent: "alpha".to_owned(),
16978                branch: "magi/x/A".to_owned(),
16979                worktree: wt_a.clone(),
16980                summary: String::new(),
16981                stat: String::new(),
16982                files: 0,
16983                commits: 0,
16984                empty: false,
16985                failed: None,
16986                verified_noop: None,
16987                duration_ms: 0,
16988                folded: false,
16989            },
16990            Candidate {
16991                index: 1,
16992                label: 'B',
16993                agent: "beta".to_owned(),
16994                branch: "magi/x/B".to_owned(),
16995                worktree: wt_b.clone(),
16996                summary: String::new(),
16997                stat: String::new(),
16998                files: 0,
16999                commits: 0,
17000                empty: false,
17001                failed: None,
17002                verified_noop: None,
17003                duration_ms: 0,
17004                folded: false,
17005            },
17006        ];
17007        state.tally = Some(Tally {
17008            first_choice: BTreeMap::from([('A', 1)]),
17009            borda: BTreeMap::new(),
17010            winner: 'A',
17011            rankings: 1,
17012            unanimous_initial: true,
17013            deliberated: false,
17014            changed_votes: 0,
17015            unanimous_final: true,
17016            tie_break: None,
17017            judges: 1,
17018            present: 1,
17019            quorum: 1,
17020            met_quorum: true,
17021            uncontested: None,
17022        });
17023        state.status = RunStatus::Ready;
17024
17025        fold_run(&mut state, false, &crate::run::home())
17026            .await
17027            .expect("fold_run");
17028
17029        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
17030        assert!(
17031            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17032            "the unmerged winner's branch survives"
17033        );
17034        assert!(
17035            !state.candidates[0].folded,
17036            "the winner is not marked folded"
17037        );
17038
17039        assert!(!wt_b.exists(), "the loser's worktree is removed");
17040        assert!(
17041            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
17042            "the loser's branch is removed"
17043        );
17044        assert!(state.candidates[1].folded, "the loser is marked folded");
17045    }
17046
17047    #[tokio::test]
17048    #[cfg(unix)]
17049    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
17050        use std::os::fd::AsRawFd as _;
17051        use std::time::Duration;
17052
17053        let tmp = tempfile::tempdir().unwrap();
17054        let repo = tmp.path().join("repo");
17055        std::fs::create_dir_all(&repo).unwrap();
17056        init_repo(&repo);
17057        let common = repo.join(".git");
17058        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
17059        let home = tmp.path().join("home");
17060        let mut config = Config::default();
17061        config.coordination = crate::config::test_coordination();
17062        config.graph.worktree_root = Some(tmp.path().join("wt"));
17063        config.graph.timeout_resource_wait = 1;
17064        let mut state = RunState::new(
17065            repo.clone(),
17066            "main".into(),
17067            "deadbeef".into(),
17068            "task".into(),
17069            config,
17070        );
17071        let root = state.worktree_root();
17072        let candidate = root.join("cand-A");
17073        let reviewer = root.join("review-1");
17074        let branch = state.branch_for('A');
17075        git::worktree_add_branch(&repo, &candidate, &branch, "main")
17076            .await
17077            .unwrap();
17078        git::worktree_add_detached(&repo, &reviewer, "main")
17079            .await
17080            .unwrap();
17081        git::acquire_worktree_config(&repo).await.unwrap();
17082        state.enabled_worktree_config = true;
17083        state.status = RunStatus::Ready;
17084        state.candidates.push(Candidate {
17085            index: 0,
17086            label: 'A',
17087            agent: "implement".into(),
17088            branch: branch.clone(),
17089            worktree: candidate.clone(),
17090            summary: String::new(),
17091            stat: String::new(),
17092            files: 0,
17093            commits: 0,
17094            empty: false,
17095            failed: None,
17096            verified_noop: None,
17097            duration_ms: 0,
17098            folded: false,
17099        });
17100        state.save_under(&home).unwrap();
17101        let config_before = std::fs::read(common.join("config")).unwrap();
17102        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
17103            .await
17104            .unwrap();
17105        // A separate OS handle bypasses the launcher's local gate, exactly as
17106        // an external updater does, and must still prevent all folding work.
17107        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
17108        assert_eq!(
17109            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
17110            0
17111        );
17112        let mut fold = Box::pin(fold_run(&mut state, true, &home));
17113        assert!(
17114            tokio::time::timeout(Duration::from_millis(200), &mut fold)
17115                .await
17116                .is_err()
17117        );
17118        assert!(candidate.exists());
17119        assert!(reviewer.exists());
17120        assert!(git::branch_exists(&repo, &branch).await.unwrap());
17121        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17122        assert_eq!(
17123            git::git(&repo, &["worktree", "list", "--porcelain"])
17124                .await
17125                .unwrap(),
17126            worktrees_before
17127        );
17128        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
17129            .await
17130            .unwrap()
17131            .unwrap_err();
17132        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
17133        drop(fold);
17134        assert!(candidate.exists() && reviewer.exists());
17135        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
17136        assert_eq!(
17137            state.resource_block.as_ref().unwrap().kind,
17138            "repository coordination"
17139        );
17140        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17141        drop(external);
17142        let mut fold = Box::pin(fold_run(&mut state, true, &home));
17143        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
17144            .await
17145            .unwrap()
17146            .unwrap();
17147        drop(fold);
17148        assert!(removed.contains(&branch));
17149        assert!(!root.exists());
17150        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
17151        assert!(state.candidates[0].folded);
17152        assert!(!state.enabled_worktree_config);
17153        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
17154        let enabled = git::git_raw(
17155            &repo,
17156            &["config", "--local", "--get", "extensions.worktreeConfig"],
17157        )
17158        .await
17159        .unwrap();
17160        assert!(!enabled.ok(), "the last config reference was released");
17161    }
17162
17163    /// A branch handed to a later run is that run's (and its pull request's):
17164    /// folding the run that released it must not delete it.
17165    #[tokio::test]
17166    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
17167        let tmp = tempfile::tempdir().expect("tempdir");
17168        let repo = tmp.path().join("repo");
17169        std::fs::create_dir_all(&repo).unwrap();
17170        init_repo(&repo);
17171        let home = tmp.path().join("home");
17172
17173        let mut config = Config::default();
17174        config.graph.worktree_root = Some(tmp.path().join("wt"));
17175        let mut state = RunState::new(
17176            repo.clone(),
17177            "main".to_owned(),
17178            "deadbeef".to_owned(),
17179            "task".to_owned(),
17180            config,
17181        );
17182        // The worktree is already gone (released); the branch survives.
17183        git::git(&repo, &["branch", "magi/x/A", "main"])
17184            .await
17185            .expect("branch");
17186        state.candidates = vec![Candidate {
17187            index: 0,
17188            label: 'A',
17189            agent: "alpha".to_owned(),
17190            branch: "magi/x/A".to_owned(),
17191            worktree: state.worktree_root().join("cand-A"),
17192            summary: String::new(),
17193            stat: String::new(),
17194            files: 0,
17195            commits: 0,
17196            empty: false,
17197            failed: None,
17198            verified_noop: None,
17199            duration_ms: 0,
17200            folded: true,
17201        }];
17202        state.released_to = Some("20260901-000000-new1".to_owned());
17203        state.released_branches = vec!["magi/x/A".to_owned()];
17204
17205        fold_run(&mut state, true, &home).await.expect("fold_run");
17206        state.status = RunStatus::Merged;
17207        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
17208        state.merge = Some(crate::run::MergeOutcome {
17209            mode: MergeMode::Pr,
17210            ok: true,
17211            detail: "confirmed merged".into(),
17212            empty: false,
17213        });
17214        fold_run(&mut state, true, &home)
17215            .await
17216            .expect("merged fold");
17217
17218        assert!(
17219            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17220            "the handed-over branch survives a fold"
17221        );
17222    }
17223
17224    /// A winner with nothing ahead of the base is caught before `gh` is ever
17225    /// asked for a pull request, and the message carries what the task's
17226    /// references resolved to.
17227    #[tokio::test]
17228    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
17229        let tmp = tempfile::tempdir().expect("tempdir");
17230        let repo = tmp.path().join("repo");
17231        std::fs::create_dir_all(&repo).unwrap();
17232        init_repo(&repo);
17233        let run = |args: &[&str]| {
17234            let out = std::process::Command::new("git")
17235                .quiet()
17236                .args(args)
17237                .current_dir(&repo)
17238                .output()
17239                .expect("spawn git");
17240            assert!(out.status.success(), "git {args:?}");
17241        };
17242        run(&["branch", "magi/x/A"]);
17243        run(&["checkout", "-q", "-b", "magi/x/B"]);
17244        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
17245        run(&["add", "-A"]);
17246        run(&["commit", "-q", "-m", "work"]);
17247        run(&["checkout", "-q", "main"]);
17248
17249        // A pull request is compared against the remote's base, so the
17250        // fixture needs one. Before it exists nothing can be read, and the
17251        // answer must be "not empty".
17252        let probe = RunState::new(
17253            repo.clone(),
17254            "main".to_owned(),
17255            "deadbeef".to_owned(),
17256            "task".to_owned(),
17257            Config::default(),
17258        );
17259        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
17260        let bare = tmp.path().join("origin.git");
17261        let out = std::process::Command::new("git")
17262            .quiet()
17263            .args(["init", "-q", "--bare"])
17264            .arg(&bare)
17265            .output()
17266            .expect("spawn git");
17267        assert!(out.status.success(), "git init --bare");
17268        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
17269        run(&["push", "-q", "origin", "main"]);
17270
17271        let mut state = RunState::new(
17272            repo.clone(),
17273            "main".to_owned(),
17274            "deadbeef".to_owned(),
17275            "task".to_owned(),
17276            Config::default(),
17277        );
17278        state.seeds = vec![refs::Seed {
17279            token: "magi/27b2/A".to_owned(),
17280            kind: refs::SeedKind::Unresolved,
17281            sha: String::new(),
17282            branch: true,
17283            detail: "no branch or commit named magi/27b2/A".to_owned(),
17284        }];
17285
17286        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
17287        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
17288        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
17289        let detail = empty_candidate_detail(&state, "main");
17290        assert!(detail.starts_with("empty candidate"), "{detail}");
17291        assert!(!detail.contains("magi/27b2/A"), "{detail}");
17292        assert!(detail.contains("--start-from"), "{detail}");
17293    }
17294
17295    /// `status == Ready` used to be read as "this is the harmless
17296    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
17297    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
17298    /// run's PR was closed without merging — and reentering `merge` with
17299    /// `mode` still `Pr` does not know the difference, so it pushed and
17300    /// opened a second pull request. `mode == Local` reproduces the same
17301    /// blind spot without a network call: reentry must not attempt another
17302    /// git merge once this node has already recorded an outcome.
17303    #[tokio::test]
17304    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
17305        let tmp = tempfile::tempdir().expect("tempdir");
17306        let repo = tmp.path().join("repo");
17307        std::fs::create_dir_all(&repo).unwrap();
17308        init_repo(&repo);
17309
17310        let mut config = Config::default();
17311        config.merge.mode = MergeMode::Local;
17312
17313        let mut state = RunState::new(
17314            repo.clone(),
17315            "main".to_owned(),
17316            "deadbeef".to_owned(),
17317            "task".to_owned(),
17318            config,
17319        );
17320        state.candidates = vec![Candidate {
17321            index: 0,
17322            label: 'A',
17323            agent: "alpha".to_owned(),
17324            branch: "does-not-exist".to_owned(),
17325            worktree: repo.clone(),
17326            summary: String::new(),
17327            stat: String::new(),
17328            files: 0,
17329            commits: 0,
17330            empty: false,
17331            failed: None,
17332            verified_noop: None,
17333            duration_ms: 0,
17334            folded: false,
17335        }];
17336        state.tally = Some(Tally {
17337            first_choice: BTreeMap::from([('A', 1)]),
17338            borda: BTreeMap::new(),
17339            winner: 'A',
17340            rankings: 1,
17341            unanimous_initial: true,
17342            deliberated: false,
17343            changed_votes: 0,
17344            unanimous_final: true,
17345            tie_break: None,
17346            judges: 0,
17347            present: 0,
17348            quorum: 0,
17349            met_quorum: true,
17350            uncontested: Some("only candidate A produced a change".to_owned()),
17351        });
17352        state.reviews = vec![ReviewRound {
17353            round: 1,
17354            head: "deadbeef".to_owned(),
17355            verified_head: None,
17356            verified_at: None,
17357            reviews: Vec::new(),
17358            e2e: Vec::new(),
17359            fix: None,
17360            blocking: 0,
17361            answered: 0,
17362            expected: 0,
17363            clean: true,
17364            verify_retried: false,
17365            e2e_deferred: false,
17366            e2e_defer_reason: None,
17367            progressed: false,
17368            vote_split: false,
17369            reconsideration: Vec::new(),
17370            verdict: None,
17371        }];
17372        state.gate = vec![CommandOutcome {
17373            command: "test".to_owned(),
17374            code: Some(0),
17375            output_tail: String::new(),
17376            duration_ms: 0,
17377            resource_blocked: false,
17378        }];
17379        state.gate_ran = true;
17380        // Reached its conclusion already — e.g. `land` closing the PR without
17381        // merging it, which (like the honest `MergeMode::None` path) leaves
17382        // `status` at `Ready`. The recorded outcome is what actually marks
17383        // this node done.
17384        state.status = RunStatus::Ready;
17385        state.merge = Some(MergeOutcome {
17386            mode: MergeMode::Local,
17387            ok: false,
17388            detail: "already concluded".to_owned(),
17389            empty: false,
17390        });
17391
17392        let mut runner = Runner {
17393            state,
17394            roles: ResolvedRoles {
17395                implementers: Vec::new(),
17396                judges: Vec::new(),
17397                reviewers: Vec::new(),
17398                fixer: None,
17399                conductor: conductor(),
17400                implementer_roster: Vec::new(),
17401                judge_roster: Vec::new(),
17402                reviewer_roster: Vec::new(),
17403            },
17404            sem: Arc::new(Semaphore::new(1)),
17405            pause: Pause::new(),
17406            interrupt: Pause::new(),
17407        };
17408
17409        runner.merge().await.expect("merge");
17410
17411        assert_eq!(
17412            runner.state.status,
17413            RunStatus::Ready,
17414            "a concluded run's status must not change on reentry"
17415        );
17416        assert_eq!(
17417            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17418            Some("already concluded"),
17419            "merge must not run again once the node already recorded an outcome"
17420        );
17421    }
17422
17423    /// `gate` leaves `state.gate_ran` false both before it has ever run and
17424    /// when its last attempt was resource-blocked (the shared build cache
17425    /// could not be acquired or confirmed fresh in time - see
17426    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
17427    /// `Vec` this also leaves behind used to read as "nothing failed" and let
17428    /// a run merge a tree the gate never actually checked - exactly the case
17429    /// a contended cache produces on every retry until it clears. `merge`
17430    /// must refuse until `gate` has actually recorded an attempt.
17431    #[tokio::test]
17432    async fn merge_refuses_a_gate_that_has_not_actually_run() {
17433        let tmp = tempfile::tempdir().expect("tempdir");
17434        let repo = tmp.path().join("repo");
17435        std::fs::create_dir_all(&repo).unwrap();
17436        init_repo(&repo);
17437
17438        let mut config = Config::default();
17439        config.merge.mode = MergeMode::Local;
17440
17441        let mut state = RunState::new(
17442            repo.clone(),
17443            "main".to_owned(),
17444            "deadbeef".to_owned(),
17445            "task".to_owned(),
17446            config,
17447        );
17448        state.candidates = vec![Candidate {
17449            index: 0,
17450            label: 'A',
17451            agent: "alpha".to_owned(),
17452            branch: "does-not-exist".to_owned(),
17453            worktree: repo.clone(),
17454            summary: String::new(),
17455            stat: String::new(),
17456            files: 0,
17457            commits: 0,
17458            empty: false,
17459            failed: None,
17460            verified_noop: None,
17461            duration_ms: 0,
17462            folded: false,
17463        }];
17464        state.tally = Some(Tally {
17465            first_choice: BTreeMap::from([('A', 1)]),
17466            borda: BTreeMap::new(),
17467            winner: 'A',
17468            rankings: 1,
17469            unanimous_initial: true,
17470            deliberated: false,
17471            changed_votes: 0,
17472            unanimous_final: true,
17473            tie_break: None,
17474            judges: 0,
17475            present: 0,
17476            quorum: 0,
17477            met_quorum: true,
17478            uncontested: Some("only candidate A produced a change".to_owned()),
17479        });
17480        state.reviews = vec![ReviewRound {
17481            round: 1,
17482            head: "deadbeef".to_owned(),
17483            verified_head: None,
17484            verified_at: None,
17485            reviews: Vec::new(),
17486            e2e: Vec::new(),
17487            fix: None,
17488            blocking: 0,
17489            answered: 0,
17490            expected: 0,
17491            clean: true,
17492            verify_retried: false,
17493            e2e_deferred: false,
17494            e2e_defer_reason: None,
17495            progressed: false,
17496            vote_split: false,
17497            reconsideration: Vec::new(),
17498            verdict: None,
17499        }];
17500        // The point: `gate` has not recorded anything yet.
17501        state.gate = Vec::new();
17502        state.gate_ran = false;
17503        state.status = RunStatus::Gating;
17504
17505        let mut runner = Runner {
17506            state,
17507            roles: ResolvedRoles {
17508                implementers: Vec::new(),
17509                judges: Vec::new(),
17510                reviewers: Vec::new(),
17511                fixer: None,
17512                conductor: conductor(),
17513                implementer_roster: Vec::new(),
17514                judge_roster: Vec::new(),
17515                reviewer_roster: Vec::new(),
17516            },
17517            sem: Arc::new(Semaphore::new(1)),
17518            pause: Pause::new(),
17519            interrupt: Pause::new(),
17520        };
17521
17522        runner.merge().await.expect("merge");
17523
17524        assert!(
17525            runner.state.merge.is_none(),
17526            "an empty gate must never be read as a passing one: {:?}",
17527            runner.state.merge
17528        );
17529    }
17530
17531    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
17532    /// commands configured and `merge.mode` is `none` (a review-only run).
17533    /// `gate` must still record a real attempt — zero commands, vacuously
17534    /// passed — rather than leaving `state.gate` empty in a way `merge`
17535    /// cannot tell apart from "never ran"; otherwise the run reaches
17536    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
17537    #[tokio::test]
17538    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
17539        let tmp = tempfile::tempdir().expect("tempdir");
17540        let repo = tmp.path().join("repo");
17541        std::fs::create_dir_all(&repo).unwrap();
17542        init_repo(&repo);
17543
17544        // Default config: `verify.gate` empty, `merge.mode` is `none`.
17545        let config = Config::default();
17546
17547        let mut state = RunState::new(
17548            repo.clone(),
17549            "main".to_owned(),
17550            "deadbeef".to_owned(),
17551            "task".to_owned(),
17552            config,
17553        );
17554        state.candidates = vec![Candidate {
17555            index: 0,
17556            label: 'A',
17557            agent: "alpha".to_owned(),
17558            branch: "does-not-exist".to_owned(),
17559            worktree: repo.clone(),
17560            summary: String::new(),
17561            stat: String::new(),
17562            files: 0,
17563            commits: 0,
17564            empty: false,
17565            failed: None,
17566            verified_noop: None,
17567            duration_ms: 0,
17568            folded: false,
17569        }];
17570        state.tally = Some(Tally {
17571            first_choice: BTreeMap::from([('A', 1)]),
17572            borda: BTreeMap::new(),
17573            winner: 'A',
17574            rankings: 1,
17575            unanimous_initial: true,
17576            deliberated: false,
17577            changed_votes: 0,
17578            unanimous_final: true,
17579            tie_break: None,
17580            judges: 0,
17581            present: 0,
17582            quorum: 0,
17583            met_quorum: true,
17584            uncontested: Some("only candidate A produced a change".to_owned()),
17585        });
17586        state.reviews = vec![ReviewRound {
17587            round: 1,
17588            head: "deadbeef".to_owned(),
17589            verified_head: None,
17590            verified_at: None,
17591            reviews: Vec::new(),
17592            e2e: Vec::new(),
17593            fix: None,
17594            blocking: 0,
17595            answered: 0,
17596            expected: 0,
17597            clean: true,
17598            verify_retried: false,
17599            e2e_deferred: false,
17600            e2e_defer_reason: None,
17601            progressed: false,
17602            vote_split: false,
17603            reconsideration: Vec::new(),
17604            verdict: None,
17605        }];
17606
17607        let mut runner = Runner {
17608            state,
17609            roles: ResolvedRoles {
17610                implementers: Vec::new(),
17611                judges: Vec::new(),
17612                reviewers: Vec::new(),
17613                fixer: None,
17614                conductor: conductor(),
17615                implementer_roster: Vec::new(),
17616                judge_roster: Vec::new(),
17617                reviewer_roster: Vec::new(),
17618            },
17619            sem: Arc::new(Semaphore::new(1)),
17620            pause: Pause::new(),
17621            interrupt: Pause::new(),
17622        };
17623
17624        runner.gate().await.expect("gate");
17625        assert!(
17626            runner.state.gate_ran,
17627            "zero configured commands is still a real attempt, not an unrun gate"
17628        );
17629        assert!(runner.state.gate.is_empty());
17630        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
17631        assert_ne!(
17632            runner.state.status,
17633            RunStatus::Blocked,
17634            "a gate with nothing to check must not read as failed"
17635        );
17636
17637        runner.merge().await.expect("merge");
17638        assert_eq!(
17639            runner.state.status,
17640            RunStatus::Ready,
17641            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
17642        );
17643    }
17644
17645    /// `Config::cache_dir` is derived from `verify.e2e` as well as
17646    /// `verify.gate` (so the e2e leg and the final gate never build against
17647    /// different directories). With zero `verify.gate` commands but a
17648    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
17649    /// that lease before discovering it had nothing to run - so a repo with
17650    /// no gate commands could come back `resource_blocked` (and therefore
17651    /// still `gate_ran == false`) on nothing but an unrelated run holding the
17652    /// cache, exactly the contention this run's own zero commands could
17653    /// never have touched. `gate` must recognise there is nothing to check
17654    /// before it ever asks for the lease.
17655    #[tokio::test]
17656    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
17657        crate::run::pin_test_home();
17658        let home = crate::run::home();
17659
17660        let tmp = tempfile::tempdir().expect("tempdir");
17661        let repo = tmp.path().join("repo");
17662        std::fs::create_dir_all(&repo).unwrap();
17663        init_repo(&repo);
17664        // Unique to this test, so holding its lease cannot collide with
17665        // another test sharing the same process-wide `home`.
17666        let cache_dir = tmp.path().join("target");
17667
17668        let mut config = Config::default();
17669        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
17670        // `verify.gate` stays empty (the default). Bounded so a regression
17671        // that does start waiting fails the test in seconds, not hangs it.
17672        config.graph.timeout_verify = Some(2);
17673        config.graph.timeout_resource_wait = 2;
17674
17675        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17676        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17677            .expect("no io error acquiring directly")
17678        {
17679            crate::cache::AcquireOutcome::Acquired(g) => g,
17680            crate::cache::AcquireOutcome::Busy(b) => {
17681                panic!("expected the direct acquire to win the lease first: {b:?}")
17682            }
17683        };
17684
17685        let mut state = RunState::new(
17686            repo.clone(),
17687            "main".to_owned(),
17688            "deadbeef".to_owned(),
17689            "task".to_owned(),
17690            config,
17691        );
17692        state.candidates = vec![Candidate {
17693            index: 0,
17694            label: 'A',
17695            agent: "alpha".to_owned(),
17696            branch: "does-not-exist".to_owned(),
17697            worktree: repo.clone(),
17698            summary: String::new(),
17699            stat: String::new(),
17700            files: 0,
17701            commits: 0,
17702            empty: false,
17703            failed: None,
17704            verified_noop: None,
17705            duration_ms: 0,
17706            folded: false,
17707        }];
17708        state.tally = Some(Tally {
17709            first_choice: BTreeMap::from([('A', 1)]),
17710            borda: BTreeMap::new(),
17711            winner: 'A',
17712            rankings: 1,
17713            unanimous_initial: true,
17714            deliberated: false,
17715            changed_votes: 0,
17716            unanimous_final: true,
17717            tie_break: None,
17718            judges: 0,
17719            present: 0,
17720            quorum: 0,
17721            met_quorum: true,
17722            uncontested: Some("only candidate A produced a change".to_owned()),
17723        });
17724        state.reviews = vec![ReviewRound {
17725            round: 1,
17726            head: "deadbeef".to_owned(),
17727            verified_head: None,
17728            verified_at: None,
17729            reviews: Vec::new(),
17730            e2e: Vec::new(),
17731            fix: None,
17732            blocking: 0,
17733            answered: 0,
17734            expected: 0,
17735            clean: true,
17736            verify_retried: false,
17737            e2e_deferred: false,
17738            e2e_defer_reason: None,
17739            progressed: false,
17740            vote_split: false,
17741            reconsideration: Vec::new(),
17742            verdict: None,
17743        }];
17744
17745        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
17746        state.reviews[0].e2e = vec![CommandOutcome {
17747            command: state.config.verify.e2e[0].clone(),
17748            code: Some(0),
17749            output_tail: String::new(),
17750            duration_ms: 0,
17751            resource_blocked: false,
17752        }];
17753        let mut runner = Runner {
17754            state,
17755            roles: ResolvedRoles {
17756                implementers: Vec::new(),
17757                judges: Vec::new(),
17758                reviewers: Vec::new(),
17759                fixer: None,
17760                conductor: conductor(),
17761                implementer_roster: Vec::new(),
17762                judge_roster: Vec::new(),
17763                reviewer_roster: Vec::new(),
17764            },
17765            sem: Arc::new(Semaphore::new(1)),
17766            pause: Pause::new(),
17767            interrupt: Pause::new(),
17768        };
17769
17770        let started = std::time::Instant::now();
17771        runner.gate().await.expect("gate");
17772        assert!(
17773            started.elapsed() < Duration::from_secs(1),
17774            "a gate with nothing to run must never wait on a lease it never needed"
17775        );
17776        assert!(
17777            runner.state.gate_ran,
17778            "zero commands is still a real, immediate attempt"
17779        );
17780        assert!(runner.state.gate.is_empty());
17781        assert_ne!(
17782            runner.state.status,
17783            RunStatus::Blocked,
17784            "must not read as resource-blocked on a lease it never asked for"
17785        );
17786    }
17787
17788    /// A separate process carries a live worker's environment throughout all
17789    /// verification entry points; the parallel harness never changes its env.
17790    #[cfg(unix)]
17791    #[tokio::test]
17792    async fn verification_is_standalone_inside_a_live_worker() {
17793        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
17794        if std::env::var_os(CHILD).is_none() {
17795            let home = tempfile::tempdir().unwrap();
17796            let generation = crate::worker::pin_current(home.path()).unwrap();
17797            let output = std::process::Command::new(std::env::current_exe().unwrap())
17798                .args([
17799                    "--exact",
17800                    "graph::tests::verification_is_standalone_inside_a_live_worker",
17801                    "--nocapture",
17802                ])
17803                .env(CHILD, "1")
17804                .env("MAGI_HOME", home.path())
17805                .env("MAGI_WORKER_HOME", home.path())
17806                .env("MAGI_WORKER_ID", "abcdef")
17807                .env("MAGI_WORKER_GENERATION", &generation.id)
17808                .env("MAGI_WORKER_EXE", &generation.executable)
17809                .env("MAGI_RUN", "verification-parent")
17810                .env("MAGI_NODE", "gate")
17811                .env("MAGI_SEAT", "worker-fixture")
17812                .env("MAGI_PROCESS_OWNER", "fixture-owner")
17813                .env("MAGI_NO_AUTOUPDATE", "0")
17814                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
17815                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
17816                .env(
17817                    "VERIFY_EXPECT_TOOLCHAIN",
17818                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
17819                )
17820                .env(
17821                    "VERIFY_EXPECT_CACHE",
17822                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
17823                )
17824                .output()
17825                .unwrap();
17826            assert!(
17827                output.status.success()
17828                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17829                "{}\n{}",
17830                String::from_utf8_lossy(&output.stdout),
17831                String::from_utf8_lossy(&output.stderr)
17832            );
17833            return;
17834        }
17835        let home = crate::run::home();
17836        let directory = home.join("workers/jobs/abcdef");
17837        std::fs::create_dir_all(&directory).unwrap();
17838        let job = crate::worker::Job {
17839            id: "abcdef".into(),
17840            sequence: 1,
17841            kind: "task".into(),
17842            key: "task:verification".into(),
17843            payload: serde_json::json!({}),
17844            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
17845            version: Some(env!("CARGO_PKG_VERSION").into()),
17846            status: crate::worker::Status::Running,
17847            result: None,
17848            pid: Some(std::process::id()),
17849            created_at: 0,
17850            finished_at: None,
17851        };
17852        std::fs::write(
17853            directory.join("job.json"),
17854            serde_json::to_vec(&job).unwrap(),
17855        )
17856        .unwrap();
17857        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
17858        let assert_worker = || {
17859            assert_eq!(
17860                crate::worker::current_job(&home).unwrap().unwrap().id,
17861                job.id
17862            );
17863            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
17864            assert_eq!(
17865                crate::worker::helper_executable(&home).unwrap(),
17866                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
17867            );
17868            assert!(
17869                crate::updater::Checker::new(&crate::config::Update {
17870                    mode: crate::config::UpdateMode::Notify,
17871                    interval: None,
17872                    auto_restart: false,
17873                })
17874                .is_none(),
17875                "the driver remains pinned, even while tests are standalone"
17876            );
17877        };
17878        assert_worker();
17879        let repo = home.join("repo");
17880        std::fs::create_dir_all(&repo).unwrap();
17881        init_repo(&repo);
17882        // Keep storage, seat/control context, containment and tool lookup.
17883        // No inherited values (or credentials) are printed on success.
17884        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
17885            test -z "${MAGI_WORKER_GENERATION+x}" &&
17886            test -z "${MAGI_WORKER_HOME+x}" &&
17887            test -z "${MAGI_WORKER_EXE+x}" &&
17888            test "$MAGI_HOME/repo" -ef . &&
17889            test "$MAGI_RUN" = verification-parent &&
17890            test "$MAGI_NODE" = gate &&
17891            test "$MAGI_SEAT" = worker-fixture &&
17892            test -n "$MAGI_PROCESS_OWNER" &&
17893            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
17894            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
17895            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
17896            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
17897        // Negative control: an ordinary child inherits the contamination.
17898        assert!(
17899            !std::process::Command::new("sh")
17900                .args(["-c", probe])
17901                .current_dir(&repo)
17902                .status()
17903                .unwrap()
17904                .success()
17905        );
17906        let unrelated_home = tempfile::tempdir().unwrap();
17907        assert!(
17908            crate::worker::current_job(unrelated_home.path()).is_err(),
17909            "an inherited worker id cannot be resolved in a unit fixture's home"
17910        );
17911        let mut runner = runner_at(RunStatus::Gating);
17912        runner.state.repo = repo.clone();
17913        runner.state.config.disk.min_free_bytes = 0;
17914        runner.state.config.coordination = crate::config::test_coordination();
17915        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
17916        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
17917        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
17918        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
17919        let winner = Candidate {
17920            index: 0,
17921            label: 'A',
17922            agent: "fixture".into(),
17923            branch: "main".into(),
17924            worktree: repo.clone(),
17925            summary: String::new(),
17926            stat: String::new(),
17927            files: 0,
17928            commits: 0,
17929            empty: false,
17930            failed: None,
17931            verified_noop: None,
17932            duration_ms: 0,
17933            folded: false,
17934        };
17935        let (e2e, retried, pids) = run_e2e_with_retry(
17936            &mut runner.state,
17937            &["sh".into(), "-c".into()],
17938            &[format!("{probe} && printf initial")],
17939            &repo,
17940            Duration::from_secs(10),
17941            "initial",
17942        )
17943        .await;
17944        assert!(!retried && pids.is_empty());
17945        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
17946        assert_eq!(e2e[0].output_tail, "initial");
17947        let gate = runner.run_gate(&winner).await.unwrap();
17948        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
17949        // An interrupted repair must recheck e2e from saved state. Landing
17950        // reuses that state and must run both pre-gate and gate again.
17951        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
17952            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
17953            passed: false,
17954            outcomes: Vec::new(),
17955        });
17956        runner.state.save().unwrap();
17957        let mut resumed = RunState::load(&runner.state.id).unwrap();
17958        let wait = crate::repo_lock::WaitPolicy::default();
17959        assert!(matches!(
17960            check_gate_repair_e2e(&mut resumed, &winner, &wait)
17961                .await
17962                .unwrap(),
17963            GateFix::Retry
17964        ));
17965        let saved = RunState::load(&resumed.id).unwrap();
17966        let repair = saved.gate_repair_e2e.unwrap();
17967        assert!(repair.passed);
17968        assert_eq!(repair.outcomes.len(), 1);
17969        assert!(repair.outcomes[0].ok());
17970        let landing = run_landing_gate(&mut resumed, &winner, &wait)
17971            .await
17972            .unwrap();
17973        assert_eq!(landing.len(), 1);
17974        assert!(landing[0].ok(), "{landing:?}");
17975        assert_eq!(
17976            std::fs::read_to_string(repo.join("phases")).unwrap(),
17977            "pre\ngate\ne2e\npre\ngate\n"
17978        );
17979        assert_worker();
17980    }
17981
17982    #[cfg(unix)]
17983    #[tokio::test]
17984    async fn verification_keeps_the_panic_and_the_full_output() {
17985        crate::run::pin_test_home();
17986        let tmp = tempfile::tempdir().unwrap();
17987        let mut config = Config::default();
17988        config.disk.min_free_bytes = 0;
17989        let mut state = RunState::new(
17990            tmp.path().to_owned(),
17991            "main".into(),
17992            "base".into(),
17993            "verification evidence".into(),
17994            config,
17995        );
17996        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
17997                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
17998                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
17999        let mut names = Vec::new();
18000        for _ in 0..2 {
18001            let (outcomes, _) = run_commands(
18002                &mut state,
18003                "gate",
18004                "gate",
18005                0,
18006                &["sh".into(), "-c".into()],
18007                &[script.into()],
18008                tmp.path(),
18009                Duration::from_secs(60),
18010            )
18011            .await;
18012            let tail = &outcomes[0].output_tail;
18013            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
18014            assert!(tail.contains("boom assertion"));
18015            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
18016            let line = tail.lines().next().unwrap();
18017            names.push(line.to_owned());
18018        }
18019        assert_ne!(names[0], names[1], "the second run must not overwrite");
18020        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
18021        assert!(full.len() > 500_000 && full.contains("boom assertion"));
18022        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
18023
18024        // Storage failures are evidence too; preserve the command's verdict.
18025        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
18026        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
18027        let (outcomes, _) = run_commands(
18028            &mut state,
18029            "gate",
18030            "gate",
18031            0,
18032            &["sh".into(), "-c".into()],
18033            &["echo failure-output; exit 7".into()],
18034            tmp.path(),
18035            Duration::from_secs(10),
18036        )
18037        .await;
18038        assert_eq!(outcomes[0].code, Some(7));
18039        assert!(state.events.iter().any(|e| {
18040            e.node == "gate" && e.message.contains("could not save the full gate output:")
18041        }));
18042    }
18043
18044    #[cfg(unix)]
18045    #[tokio::test]
18046    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
18047        crate::run::pin_test_home();
18048        let tmp = tempfile::tempdir().unwrap();
18049        let mut config = Config::default();
18050        config.disk.min_free_bytes = 0;
18051        let mut state = RunState::new(
18052            tmp.path().to_owned(),
18053            "main".into(),
18054            "base".into(),
18055            "verification evidence".into(),
18056            config,
18057        );
18058        let (outcomes, pids) = run_commands(
18059            &mut state,
18060            "gate",
18061            "gate",
18062            1,
18063            &["sh".into(), "-c".into()],
18064            &[
18065                "printf before-timeout; exec sleep 30".into(),
18066                "kill -TERM $$".into(),
18067                "exit 7".into(),
18068                "printf success".into(),
18069            ],
18070            tmp.path(),
18071            Duration::from_secs(1),
18072        )
18073        .await;
18074        wait_for_timed_out_children_to_die(&pids).await;
18075        assert_eq!(
18076            pids.len(),
18077            1,
18078            "unleased timeout retains its death-check PID"
18079        );
18080        assert_eq!(outcomes.len(), 4);
18081        assert_eq!(outcomes[0].code, None);
18082        assert!(outcomes[0].output_tail.contains("before-timeout"));
18083        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
18084        assert_eq!(outcomes[1].code, None);
18085        assert!(outcomes[1].output_tail.contains("command terminated:"));
18086        assert!(!outcomes[1].output_tail.contains("timed out"));
18087        assert_eq!(outcomes[2].code, Some(7));
18088        assert!(!outcomes[2].output_tail.contains("timed out"));
18089        assert_eq!(outcomes[3].code, Some(0));
18090        assert_eq!(outcomes[3].output_tail, "success");
18091        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
18092    }
18093
18094    /// The addendum's second gap: a `verify.gate` command running for real
18095    /// wall-clock time had nothing at all to show for it in `active` before
18096    /// `run_commands` learned to record it — a run could sit in `Gating` for
18097    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
18098    /// what was actually happening. Proven with a genuinely still-running
18099    /// command, not just a before/after check on the final state: a poller
18100    /// task reads the same `run.json` `gate()` is writing, the same way the
18101    /// phone or `magi show` would, while the shell command is still blocked
18102    /// on its own release marker.
18103    #[tokio::test]
18104    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
18105        crate::run::pin_test_home();
18106
18107        let tmp = tempfile::tempdir().expect("tempdir");
18108        let repo = tmp.path().join("repo");
18109        std::fs::create_dir_all(&repo).unwrap();
18110        init_repo(&repo);
18111
18112        let mut config = Config::default();
18113        config.verify.gate = vec![
18114            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
18115             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
18116                .to_owned(),
18117        ];
18118
18119        let mut state = RunState::new(
18120            repo.clone(),
18121            "main".to_owned(),
18122            "deadbeef".to_owned(),
18123            "task".to_owned(),
18124            config,
18125        );
18126        let run_id = state.id.clone();
18127        state.candidates = vec![Candidate {
18128            index: 0,
18129            label: 'A',
18130            agent: "alpha".to_owned(),
18131            branch: "does-not-exist".to_owned(),
18132            worktree: repo.clone(),
18133            summary: String::new(),
18134            stat: String::new(),
18135            files: 0,
18136            commits: 0,
18137            empty: false,
18138            failed: None,
18139            verified_noop: None,
18140            duration_ms: 0,
18141            folded: false,
18142        }];
18143        state.tally = Some(Tally {
18144            first_choice: BTreeMap::from([('A', 1)]),
18145            borda: BTreeMap::new(),
18146            winner: 'A',
18147            rankings: 1,
18148            unanimous_initial: true,
18149            deliberated: false,
18150            changed_votes: 0,
18151            unanimous_final: true,
18152            tie_break: None,
18153            judges: 0,
18154            present: 0,
18155            quorum: 0,
18156            met_quorum: true,
18157            uncontested: Some("only candidate A produced a change".to_owned()),
18158        });
18159        state.reviews = vec![ReviewRound {
18160            round: 1,
18161            head: "deadbeef".to_owned(),
18162            verified_head: None,
18163            verified_at: None,
18164            reviews: Vec::new(),
18165            e2e: Vec::new(),
18166            fix: None,
18167            blocking: 0,
18168            answered: 0,
18169            expected: 0,
18170            clean: true,
18171            verify_retried: false,
18172            e2e_deferred: false,
18173            e2e_defer_reason: None,
18174            progressed: false,
18175            vote_split: false,
18176            reconsideration: Vec::new(),
18177            verdict: None,
18178        }];
18179
18180        let mut runner = Runner {
18181            state,
18182            roles: ResolvedRoles {
18183                implementers: Vec::new(),
18184                judges: Vec::new(),
18185                reviewers: Vec::new(),
18186                fixer: None,
18187                conductor: conductor(),
18188                implementer_roster: Vec::new(),
18189                judge_roster: Vec::new(),
18190                reviewer_roster: Vec::new(),
18191            },
18192            sem: Arc::new(Semaphore::new(1)),
18193            pause: Pause::new(),
18194            interrupt: Pause::new(),
18195        };
18196
18197        let started_marker = repo.join("started.marker");
18198        let release_marker = repo.join("release.marker");
18199        let poller = tokio::spawn(async move {
18200            // Bounded so a regression that never records the task entry
18201            // fails this test in seconds instead of hanging the suite —
18202            // the same shape `a_park_requested_while_a_seat_is_mid_call_
18203            // does_not_cut_it_short` uses for the same reason.
18204            for _ in 0..100 {
18205                if started_marker.exists()
18206                    && let Ok(s) = crate::run::RunState::load(&run_id)
18207                    && let Some(a) = s.active.get("gate")
18208                {
18209                    std::fs::write(&release_marker, b"go").expect("release marker");
18210                    return Some(a.clone());
18211                }
18212                tokio::time::sleep(Duration::from_millis(50)).await;
18213            }
18214            None
18215        });
18216
18217        runner.gate().await.expect("gate");
18218        let captured = poller.await.expect("poller task");
18219        let captured = captured.expect(
18220            "the poller never saw a `gate` task entry in run.json while the command was \
18221             still blocked on its own release marker",
18222        );
18223
18224        assert_eq!(captured.task.as_deref(), Some("gate"));
18225        assert_eq!(captured.node, "gate");
18226        assert_eq!(captured.index, Some(1));
18227        assert_eq!(captured.total, Some(1));
18228        assert!(
18229            captured
18230                .command
18231                .as_deref()
18232                .is_some_and(|c| c.contains("started.marker")),
18233            "{captured:?}"
18234        );
18235
18236        assert!(
18237            runner.state.active.is_empty(),
18238            "the entry must be cleared once the command actually finished: {:?}",
18239            runner.state.active
18240        );
18241        assert!(runner.state.gate_ran);
18242        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18243    }
18244
18245    /// The hand-off over a blocking finding a reviewer rejected on leaves a
18246    /// record for `land`; one with only a Minor, or no reject, leaves none.
18247    #[tokio::test]
18248    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
18249        use crate::verdict::{Finding, ReviewVote, Severity};
18250        crate::run::pin_test_home();
18251        let tmp = tempfile::tempdir().expect("tempdir");
18252        let repo = tmp.path().join("repo");
18253        std::fs::create_dir_all(&repo).unwrap();
18254        init_repo(&repo);
18255
18256        for (severity, vote, expect) in [
18257            (Severity::Major, ReviewVote::Reject, true),
18258            (Severity::Minor, ReviewVote::Reject, false),
18259            (Severity::Major, ReviewVote::Approve, false),
18260        ] {
18261            let mut round = review_round(false, 1, 1, 1, false, true);
18262            round.reviews = vec![ReviewRecord {
18263                reviewer: 1,
18264                agent: "alpha".to_owned(),
18265                summary: String::new(),
18266                findings: vec![Finding {
18267                    id: "R1-1-1".to_owned(),
18268                    severity,
18269                    file: None,
18270                    line: None,
18271                    title: "t".to_owned(),
18272                    detail: String::new(),
18273                }],
18274                vote: Some(vote),
18275                failed: None,
18276                duration_ms: 0,
18277                attempts: 0,
18278            }];
18279            let mut state = RunState::new(
18280                repo.clone(),
18281                "main".to_owned(),
18282                "deadbeef".to_owned(),
18283                "task".to_owned(),
18284                Config::default(),
18285            );
18286            state.reviews = vec![round];
18287            let mut runner = Runner {
18288                state,
18289                roles: ResolvedRoles {
18290                    implementers: Vec::new(),
18291                    judges: Vec::new(),
18292                    reviewers: Vec::new(),
18293                    fixer: None,
18294                    conductor: conductor(),
18295                    implementer_roster: Vec::new(),
18296                    judge_roster: Vec::new(),
18297                    reviewer_roster: Vec::new(),
18298                },
18299                sem: Arc::new(Semaphore::new(1)),
18300                pause: Pause::new(),
18301                interrupt: Pause::new(),
18302            };
18303            let shell = runner.state.config.shell();
18304            runner
18305                .stop_reviewing("round budget spent", &shell, &repo)
18306                .await
18307                .expect("stop_reviewing");
18308            assert_eq!(runner.state.status, RunStatus::Gating);
18309            assert_eq!(
18310                runner.state.contested_handoff.is_some(),
18311                expect,
18312                "{severity:?} + {vote:?}"
18313            );
18314        }
18315    }
18316
18317    /// The shape the incident this whole fix responds to actually had: the
18318    /// round budget spent, the last round's own e2e blocked on the shared
18319    /// build cache (held here by a live pid — this test process — exactly
18320    /// `cache`'s own unit tests' pattern for "another owner, still alive"
18321    /// without forking a process). `stop_reviewing` must retry it — not
18322    /// silently leave the round looking untouched (the catch-up-only half of
18323    /// the bug), and not read the contention as a red `e2e` and block the
18324    /// run on it (the other half). Called directly, the same way
18325    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
18326    /// above exercises `gate`, so this never needs a real cargo build to
18327    /// reach: the lease is never released, so `with_cache_lease` never gets
18328    /// past acquiring it into anything that would need a real workspace.
18329    #[tokio::test]
18330    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
18331        crate::run::pin_test_home();
18332        let home = crate::run::home();
18333
18334        let tmp = tempfile::tempdir().expect("tempdir");
18335        let repo = tmp.path().join("repo");
18336        std::fs::create_dir_all(&repo).unwrap();
18337        init_repo(&repo);
18338        let head = crate::git::rev_parse(&repo, "HEAD")
18339            .await
18340            .expect("rev-parse");
18341        // Unique to this test, so holding its lease cannot collide with
18342        // another test sharing the same process-wide `home`.
18343        let cache_dir = tmp.path().join("target");
18344
18345        let mut config = Config::default();
18346        config.verify.e2e = vec![format!(
18347            "CARGO_TARGET_DIR='{}' test -f README.md",
18348            cache_dir.display()
18349        )];
18350        config.graph.review_rounds = 1;
18351        // Bounded so a regression that does start waiting fails the test in
18352        // seconds, not hangs it.
18353        config.graph.timeout_verify = Some(2);
18354        config.graph.timeout_resource_wait = 2;
18355
18356        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18357        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18358            .expect("no io error acquiring directly")
18359        {
18360            crate::cache::AcquireOutcome::Acquired(g) => g,
18361            crate::cache::AcquireOutcome::Busy(b) => {
18362                panic!("expected the direct acquire to win the lease first: {b:?}")
18363            }
18364        };
18365
18366        let mut state = RunState::new(
18367            repo.clone(),
18368            "main".to_owned(),
18369            head.clone(),
18370            "task".to_owned(),
18371            config,
18372        );
18373        state.candidates = vec![Candidate {
18374            index: 0,
18375            label: 'A',
18376            agent: "alpha".to_owned(),
18377            branch: "does-not-exist".to_owned(),
18378            worktree: repo.clone(),
18379            summary: String::new(),
18380            stat: String::new(),
18381            files: 0,
18382            commits: 0,
18383            empty: false,
18384            failed: None,
18385            verified_noop: None,
18386            duration_ms: 0,
18387            folded: false,
18388        }];
18389        state.tally = Some(Tally {
18390            first_choice: BTreeMap::from([('A', 1)]),
18391            borda: BTreeMap::new(),
18392            winner: 'A',
18393            rankings: 1,
18394            unanimous_initial: true,
18395            deliberated: false,
18396            changed_votes: 0,
18397            unanimous_final: true,
18398            tie_break: None,
18399            judges: 0,
18400            present: 0,
18401            quorum: 0,
18402            met_quorum: true,
18403            uncontested: Some("only candidate A produced a change".to_owned()),
18404        });
18405        // The round budget's last round, deferred: `needs_catchup_run`'s
18406        // other trigger. `stop_reviewing`'s retry machinery must treat this
18407        // exactly like a resource-blocked attempt once it actually runs.
18408        state.reviews = vec![ReviewRound {
18409            round: 1,
18410            head: head.clone(),
18411            verified_head: None,
18412            verified_at: None,
18413            reviews: Vec::new(),
18414            e2e: Vec::new(),
18415            fix: None,
18416            blocking: 1,
18417            answered: 1,
18418            expected: 1,
18419            clean: false,
18420            verify_retried: false,
18421            e2e_deferred: true,
18422            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
18423            progressed: false,
18424            vote_split: false,
18425            reconsideration: Vec::new(),
18426            verdict: None,
18427        }];
18428
18429        let mut runner = Runner {
18430            state,
18431            roles: ResolvedRoles {
18432                implementers: Vec::new(),
18433                judges: Vec::new(),
18434                reviewers: Vec::new(),
18435                fixer: None,
18436                conductor: conductor(),
18437                implementer_roster: Vec::new(),
18438                judge_roster: Vec::new(),
18439                reviewer_roster: Vec::new(),
18440            },
18441            sem: Arc::new(Semaphore::new(1)),
18442            pause: Pause::new(),
18443            interrupt: Pause::new(),
18444        };
18445
18446        let shell = runner.state.config.shell();
18447        runner
18448            .stop_reviewing("round budget spent", &shell, &repo)
18449            .await
18450            .expect("stop_reviewing");
18451
18452        let last = runner.state.reviews.last().expect("round record");
18453        assert_eq!(
18454            last.e2e_status(),
18455            E2eStatus::ResourceBlocked,
18456            "the shared cache is still held; the attempt must read as blocked, not deferred or \
18457             failed: {last:?}"
18458        );
18459        assert_eq!(
18460            last.verified_head.as_deref(),
18461            Some(head.as_str()),
18462            "which commit this attempt targeted is known even though nothing finished checking \
18463             it"
18464        );
18465        let first_attempt_at = last
18466            .verified_at
18467            .expect("when this attempt ran is known too");
18468        assert_ne!(
18469            runner.state.status,
18470            RunStatus::Blocked,
18471            "contention is evidence about the machine, not the patch — it must not settle the \
18472             run as blocked: {:?}",
18473            runner.state.status
18474        );
18475        assert!(
18476            !runner
18477                .state
18478                .events
18479                .iter()
18480                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
18481            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
18482            runner.state.events
18483        );
18484
18485        // The cache is still held: a later reentry must retry the same
18486        // round's verification again — not leave it looking exactly as
18487        // untouched as the first blocked attempt, which is indistinguishable
18488        // from never having tried again at all.
18489        runner
18490            .stop_reviewing("round budget spent", &shell, &repo)
18491            .await
18492            .expect("stop_reviewing retry");
18493        assert_eq!(
18494            runner.state.reviews.len(),
18495            1,
18496            "no new round was started: {:?}",
18497            runner.state.reviews
18498        );
18499        let last = runner.state.reviews.last().expect("round record");
18500        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
18501        assert!(
18502            last.verified_at.expect("still known") > first_attempt_at,
18503            "a second reentry must be a fresh attempt, not a stale copy of the first"
18504        );
18505        assert_ne!(runner.state.status, RunStatus::Blocked);
18506
18507        held.release();
18508    }
18509
18510    #[cfg(unix)]
18511    #[tokio::test]
18512    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
18513        ask_test_home();
18514        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
18515            let tmp = tempfile::tempdir().unwrap();
18516            let repo = tmp.path().join("repo");
18517            std::fs::create_dir(&repo).unwrap();
18518            init_repo(&repo);
18519            let run_git = |args: &[&str]| {
18520                let out = std::process::Command::new("git")
18521                    .args(args)
18522                    .current_dir(&repo)
18523                    .output()
18524                    .unwrap();
18525                assert!(
18526                    out.status.success(),
18527                    "{args:?}: {}",
18528                    String::from_utf8_lossy(&out.stderr)
18529                );
18530            };
18531            let origin = tmp.path().join("origin.git");
18532            run_git(&["init", "--bare", origin.to_str().unwrap()]);
18533            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
18534            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
18535            run_git(&["add", "spec.rs"]);
18536            run_git(&["commit", "-m", "initial spec"]);
18537            run_git(&["push", "origin", "main"]);
18538            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
18539            run_git(&["checkout", "-b", "fix/review"]);
18540            std::fs::write(
18541                repo.join("check.rs"),
18542                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
18543            )
18544            .unwrap();
18545            run_git(&["add", "check.rs"]);
18546            run_git(&["commit", "-m", "use spec"]);
18547            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
18548            run_git(&["checkout", "main"]);
18549            std::fs::write(
18550                repo.join("spec.rs"),
18551                "struct AgentSpec { max_turn_secs: u64 }\n",
18552            )
18553            .unwrap();
18554            run_git(&["add", "spec.rs"]);
18555            run_git(&["commit", "-m", "extend spec"]);
18556            run_git(&["push", "origin", "main"]);
18557            run_git(&["checkout", "fix/review"]);
18558
18559            let mut runner = runner_at(RunStatus::Reviewing);
18560            runner.state.repo = repo.clone();
18561            runner.state.base_commit = base;
18562            runner.state.pr_head = Some("fix/review".into());
18563            runner.state.config.graph.review_rounds = max_rounds;
18564            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
18565            runner.state.config.graph.sessions = false;
18566            runner.state.config.disk.min_free_bytes = 0;
18567            runner.state.config.verify.e2e =
18568                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
18569            runner.state.candidates = vec![Candidate {
18570                index: 0,
18571                label: 'A',
18572                agent: "fixture".into(),
18573                branch: "fix/review".into(),
18574                worktree: repo.clone(),
18575                summary: String::new(),
18576                stat: String::new(),
18577                files: 1,
18578                commits: 1,
18579                empty: false,
18580                failed: None,
18581                verified_noop: None,
18582                duration_ms: 0,
18583                folded: false,
18584            }];
18585            runner.state.tally = Some(Tally {
18586                first_choice: BTreeMap::new(),
18587                borda: BTreeMap::new(),
18588                winner: 'A',
18589                rankings: 0,
18590                unanimous_initial: true,
18591                deliberated: false,
18592                changed_votes: 0,
18593                unanimous_final: true,
18594                tie_break: None,
18595                judges: 0,
18596                present: 0,
18597                quorum: 0,
18598                met_quorum: true,
18599                uncontested: Some("review-only".into()),
18600            });
18601            let mut last = review_round(false, 0, 1, 1, false, false);
18602            last.head = reviewed.clone();
18603            last.e2e_deferred = true;
18604            last.e2e[0].resource_blocked = true;
18605            last.e2e[0].output_tail = "waiting for the shared build cache".into();
18606            if max_rounds == 1 {
18607                last.e2e.clear();
18608            }
18609            runner.state.reviews = vec![last];
18610            let calls = tmp.path().join("fix-calls");
18611            let mut fixer = spec("repair");
18612            fixer
18613                .env
18614                .insert("FIX_CALLS".into(), calls.display().to_string());
18615            fixer.command = vec!["sh".into(), "-c".into(), r#"
18616                echo fix >> "$FIX_CALLS"
18617                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
18618                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
18619                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
18620                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
18621            "#.into()];
18622            let mut reviewer = spec("reviewer");
18623            reviewer.command = vec![
18624                "sh".into(),
18625                "-c".into(),
18626                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
18627                    .into(),
18628            ];
18629            runner.roles.fixer = Some(vec![fixer]);
18630            runner.roles.reviewers = vec![reviewer];
18631
18632            runner.sync_to_base().await.unwrap();
18633            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
18634            assert_ne!(rebased, reviewed);
18635            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
18636            if reentry {
18637                // Simulate a crash after catch-up evidence was saved but before
18638                // dispatching the fixer. No dedicated pending-repair field exists.
18639                let shell = runner.state.config.shell();
18640                runner
18641                    .stop_unresolved_review("deferred verification", &shell, &repo)
18642                    .await
18643                    .unwrap();
18644                assert!(runner.catchup_repairable());
18645                assert!(!calls.exists());
18646                runner.state = RunState::load(&runner.state.id).unwrap();
18647                assert!(runner.catchup_repairable());
18648            }
18649            runner.review_loop().await.unwrap();
18650            let first = &runner.state.reviews[0];
18651            assert_eq!(first.head, reviewed);
18652            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
18653            assert!(
18654                first.verify_retried,
18655                "a reproduced build failure gets exactly one retry: {:?}",
18656                first.e2e
18657            );
18658            assert!(first.e2e[0].output_tail.contains("E0063"));
18659            assert!(!runner.catchup_repairable());
18660            if max_rounds == 2 {
18661                assert_eq!(
18662                    runner.state.status,
18663                    RunStatus::Gating,
18664                    "{}",
18665                    crate::report::run(&runner.state)
18666                );
18667                assert_eq!(runner.state.reviews.len(), 2);
18668                assert!(first.fix.as_ref().unwrap().committed);
18669                assert!(runner.state.reviews[1].clean);
18670                assert_eq!(
18671                    runner
18672                        .state
18673                        .reviews
18674                        .iter()
18675                        .filter(|r| r.fix.is_some())
18676                        .count(),
18677                    1
18678                );
18679                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
18680                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
18681            } else {
18682                assert_eq!(runner.state.status, RunStatus::Blocked);
18683                assert!(first.fix.is_none());
18684                assert!(!calls.exists());
18685                let recovery = crate::recovery::assess(
18686                    "prior",
18687                    Some(&runner.state),
18688                    &runner.landing_base(),
18689                    crate::recovery::Relation::Diverged,
18690                );
18691                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
18692                let reasons = recovery.reasons.join("\n");
18693                assert!(
18694                    reasons.contains(&reviewed) && reasons.contains(&rebased),
18695                    "{reasons}"
18696                );
18697                assert!(
18698                    reasons.contains("base sync to")
18699                        && reasons.contains("additional review is required"),
18700                    "{reasons}"
18701                );
18702            }
18703        }
18704    }
18705
18706    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
18707    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
18708    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
18709    /// own top-of-function fast path (`review_conclusion`) correctly reads
18710    /// this shape as `None` rather than guessing `Blocked`, and the loop's
18711    /// own `for` range is empty once the round budget is spent, so
18712    /// `review_loop` must retry the check itself rather than silently doing
18713    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
18714    /// above exercises directly, but through `review_loop`'s own entry point
18715    /// this time, proving the wiring between the two rather than just the
18716    /// retry logic in isolation.
18717    #[tokio::test]
18718    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
18719        crate::run::pin_test_home();
18720        let home = crate::run::home();
18721
18722        let tmp = tempfile::tempdir().expect("tempdir");
18723        let repo = tmp.path().join("repo");
18724        std::fs::create_dir_all(&repo).unwrap();
18725        init_repo(&repo);
18726        let head = crate::git::rev_parse(&repo, "HEAD")
18727            .await
18728            .expect("rev-parse");
18729        let cache_dir = tmp.path().join("target");
18730
18731        let mut config = Config::default();
18732        config.verify.e2e = vec![format!(
18733            "CARGO_TARGET_DIR='{}' test -f README.md",
18734            cache_dir.display()
18735        )];
18736        config.graph.review_rounds = 1;
18737        config.graph.timeout_verify = Some(2);
18738        config.graph.timeout_resource_wait = 2;
18739
18740        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18741        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18742            .expect("no io error acquiring directly")
18743        {
18744            crate::cache::AcquireOutcome::Acquired(g) => g,
18745            crate::cache::AcquireOutcome::Busy(b) => {
18746                panic!("expected the direct acquire to win the lease first: {b:?}")
18747            }
18748        };
18749
18750        let mut state = RunState::new(
18751            repo.clone(),
18752            "main".to_owned(),
18753            head.clone(),
18754            "task".to_owned(),
18755            config,
18756        );
18757        state.candidates = vec![Candidate {
18758            index: 0,
18759            label: 'A',
18760            agent: "alpha".to_owned(),
18761            branch: "does-not-exist".to_owned(),
18762            worktree: repo.clone(),
18763            summary: String::new(),
18764            stat: String::new(),
18765            files: 0,
18766            commits: 0,
18767            empty: false,
18768            failed: None,
18769            verified_noop: None,
18770            duration_ms: 0,
18771            folded: false,
18772        }];
18773        state.tally = Some(Tally {
18774            first_choice: BTreeMap::from([('A', 1)]),
18775            borda: BTreeMap::new(),
18776            winner: 'A',
18777            rankings: 1,
18778            unanimous_initial: true,
18779            deliberated: false,
18780            changed_votes: 0,
18781            unanimous_final: true,
18782            tie_break: None,
18783            judges: 0,
18784            present: 0,
18785            quorum: 0,
18786            met_quorum: true,
18787            uncontested: Some("only candidate A produced a change".to_owned()),
18788        });
18789        // The exact shape a prior process's `stop_reviewing` would have left
18790        // on disk: the round budget's last round, a real attempt already
18791        // made and already resource-blocked.
18792        state.reviews = vec![ReviewRound {
18793            round: 1,
18794            head: head.clone(),
18795            verified_head: Some(head.clone()),
18796            verified_at: Some(jiff::Timestamp::now()),
18797            reviews: Vec::new(),
18798            e2e: vec![CommandOutcome {
18799                command: format!(
18800                    "CARGO_TARGET_DIR='{}' test -f README.md",
18801                    cache_dir.display()
18802                ),
18803                code: None,
18804                output_tail: "waiting for the shared build cache".to_owned(),
18805                duration_ms: 0,
18806                resource_blocked: true,
18807            }],
18808            fix: None,
18809            blocking: 1,
18810            answered: 1,
18811            expected: 1,
18812            clean: false,
18813            verify_retried: false,
18814            e2e_deferred: false,
18815            e2e_defer_reason: None,
18816            progressed: false,
18817            vote_split: false,
18818            reconsideration: Vec::new(),
18819            verdict: None,
18820        }];
18821
18822        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
18823        let mut runner = Runner {
18824            state,
18825            roles: ResolvedRoles {
18826                implementers: Vec::new(),
18827                judges: Vec::new(),
18828                reviewers: Vec::new(),
18829                fixer: None,
18830                conductor: conductor(),
18831                implementer_roster: Vec::new(),
18832                judge_roster: Vec::new(),
18833                reviewer_roster: Vec::new(),
18834            },
18835            sem: Arc::new(Semaphore::new(1)),
18836            pause: Pause::new(),
18837            interrupt: Pause::new(),
18838        };
18839
18840        // The lease is still held throughout, so this reentry's own retry is
18841        // also contended — proving `review_loop` actually tried again (not
18842        // that it happened to succeed) is what the timestamp comparison
18843        // below is for.
18844        runner.review_loop().await.expect("review_loop");
18845
18846        assert_eq!(
18847            runner.state.reviews.len(),
18848            1,
18849            "no new round was started on top of the unresolved one: {:?}",
18850            runner.state.reviews
18851        );
18852        let last = &runner.state.reviews[0];
18853        assert_eq!(
18854            last.e2e_status(),
18855            E2eStatus::ResourceBlocked,
18856            "still contended: {last:?}"
18857        );
18858        assert!(
18859            last.verified_at.expect("still known") > first_attempt_at,
18860            "review_loop must have actually retried the check, not left it exactly as found"
18861        );
18862        assert_ne!(
18863            runner.state.status,
18864            RunStatus::Blocked,
18865            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
18866            runner.state.status
18867        );
18868
18869        held.release();
18870    }
18871
18872    #[tokio::test]
18873    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
18874        crate::run::pin_test_home();
18875        for status in [RunStatus::Blocked, RunStatus::Gating] {
18876            let mut runner = runner_at(status);
18877            runner.state.config.graph.land = true;
18878            runner.state.gate_ran = true;
18879            runner.state.merge = Some(MergeOutcome {
18880                mode: MergeMode::Pr,
18881                ok: false,
18882                detail: "the fixer produced no commit".into(),
18883                empty: false,
18884            });
18885            let url = "https://example.invalid/o/r/pull/1";
18886            runner.state.pr = Some(crate::run::PrRecord {
18887                url: url.into(),
18888                number: 1,
18889                state: "open".into(),
18890                checks: "red".into(),
18891                round: 1,
18892                rounds: 1,
18893                red_at_merge: Vec::new(),
18894            });
18895            runner.state.save().unwrap();
18896            // The nonexistent repository makes observation fail before a forge
18897            // command can run. We still must reach land, rather than silently
18898            // return from the graph with its old gate and merge records.
18899            runner.execute_graph().await.unwrap();
18900            assert_eq!(runner.state.status, RunStatus::Blocked);
18901            assert!(runner.state.land_sync_pending);
18902            assert!(runner.state.events.iter().any(|event|
18903                event.node == "land" && event.message == format!("watching {url}")));
18904            assert_eq!(
18905                runner.state.merge.as_ref().unwrap().detail,
18906                "the fixer produced no commit"
18907            );
18908        }
18909    }
18910
18911    #[tokio::test]
18912    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
18913        crate::run::pin_test_home();
18914        let tmp = tempfile::tempdir().expect("tempdir");
18915        let repo = tmp.path().join("repo");
18916        std::fs::create_dir_all(&repo).unwrap();
18917        init_repo(&repo);
18918
18919        let mut config = Config::default();
18920        config.merge.mode = MergeMode::Pr;
18921        config.graph.land = true;
18922        config.graph.land_approval = false;
18923
18924        let mut state = RunState::new(
18925            repo.clone(),
18926            "main".to_owned(),
18927            "deadbeef".to_owned(),
18928            "task".to_owned(),
18929            config,
18930        );
18931        state.candidates = vec![Candidate {
18932            index: 0,
18933            label: 'A',
18934            agent: "alpha".to_owned(),
18935            branch: "does-not-exist".to_owned(),
18936            worktree: repo.clone(),
18937            summary: String::new(),
18938            stat: String::new(),
18939            files: 0,
18940            commits: 0,
18941            empty: false,
18942            failed: None,
18943            verified_noop: None,
18944            duration_ms: 0,
18945            folded: false,
18946        }];
18947        state.tally = Some(Tally {
18948            first_choice: BTreeMap::from([('A', 1)]),
18949            borda: BTreeMap::new(),
18950            winner: 'A',
18951            rankings: 1,
18952            unanimous_initial: true,
18953            deliberated: false,
18954            changed_votes: 0,
18955            unanimous_final: true,
18956            tie_break: None,
18957            judges: 0,
18958            present: 0,
18959            quorum: 0,
18960            met_quorum: true,
18961            uncontested: Some("only candidate A produced a change".to_owned()),
18962        });
18963        state.reviews = vec![ReviewRound {
18964            round: 1,
18965            head: "deadbeef".to_owned(),
18966            verified_head: None,
18967            verified_at: None,
18968            reviews: Vec::new(),
18969            e2e: Vec::new(),
18970            fix: None,
18971            blocking: 0,
18972            answered: 0,
18973            expected: 0,
18974            clean: true,
18975            verify_retried: false,
18976            e2e_deferred: false,
18977            e2e_defer_reason: None,
18978            progressed: false,
18979            vote_split: false,
18980            reconsideration: Vec::new(),
18981            verdict: None,
18982        }];
18983        state.gate = vec![CommandOutcome {
18984            command: "test".to_owned(),
18985            code: Some(0),
18986            output_tail: String::new(),
18987            duration_ms: 0,
18988            resource_blocked: false,
18989        }];
18990        state.gate_ran = true;
18991        // A first pass through `merge` already pushed and opened this pull
18992        // request; `status` is `Landing` because a previous call into `land`
18993        // parked or was interrupted before it reached a terminal outcome.
18994        state.status = RunStatus::Landing;
18995        state.merge = Some(MergeOutcome {
18996            mode: MergeMode::Pr,
18997            ok: true,
18998            detail: "https://example.invalid/x/y/pull/1".to_owned(),
18999            empty: false,
19000        });
19001
19002        // The Landing-resume shortcut calls `run_land` directly rather than
19003        // through `merge`, which is exactly the call site that used to skip
19004        // `settle_questions` - see the fixture below.
19005        ask_test_home();
19006        // A real resume loads a persisted run; driver ownership is checked
19007        // against that record before the Landing shortcut is entered.
19008        state.save().expect("save landing run");
19009        let store = ask::Questions::open();
19010        let q = ask_open_question(&store, &state.id);
19011
19012        let mut runner = Runner {
19013            state,
19014            roles: ResolvedRoles {
19015                implementers: Vec::new(),
19016                judges: Vec::new(),
19017                reviewers: Vec::new(),
19018                fixer: None,
19019                conductor: conductor(),
19020                implementer_roster: Vec::new(),
19021                judge_roster: Vec::new(),
19022                reviewer_roster: Vec::new(),
19023            },
19024            sem: Arc::new(Semaphore::new(1)),
19025            pause: Pause::new(),
19026            interrupt: Pause::new(),
19027        };
19028
19029        // `execute`, not `merge` directly: the Landing-resume shortcut lives
19030        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
19031        // exactly because `review_loop` would otherwise clobber the marker
19032        // first.
19033        runner.execute().await.expect("execute");
19034
19035        assert_eq!(
19036            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19037            Some("https://example.invalid/x/y/pull/1"),
19038            "reentry must not push again or open a second pull request over the \
19039             one `land` is already watching"
19040        );
19041        assert_ne!(
19042            runner.state.status,
19043            RunStatus::Landing,
19044            "land could not actually reach the fake pull request, so it must \
19045             have given up rather than left the run silently parked forever"
19046        );
19047        // `land` could not reach the fake pull request, so it gave up into
19048        // `Blocked` - still resumable, so the question must not have been
19049        // swept just because this branch now also calls `settle_questions`.
19050        assert_eq!(runner.state.status, RunStatus::Blocked);
19051        assert!(
19052            store.get(&q.id).unwrap().status.open(),
19053            "Blocked is still alive; settle_questions must have been a no-op here"
19054        );
19055    }
19056
19057    fn state_with_round(round: ReviewRound) -> RunState {
19058        let mut s = RunState::new(
19059            PathBuf::from("/repo"),
19060            "main".to_owned(),
19061            "abc1234".to_owned(),
19062            "add retries".to_owned(),
19063            Config::default(),
19064        );
19065        s.reviews = vec![round];
19066        s
19067    }
19068
19069    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
19070        crate::verdict::Finding {
19071            id: id.to_owned(),
19072            severity,
19073            file: None,
19074            line: None,
19075            title: title.to_owned(),
19076            detail: String::new(),
19077        }
19078    }
19079
19080    #[test]
19081    fn pr_body_names_open_findings_and_declined_ones() {
19082        let round = ReviewRound {
19083            round: 2,
19084            head: "deadbee".to_owned(),
19085            verified_head: None,
19086            verified_at: None,
19087            reviews: vec![ReviewRecord {
19088                attempts: 0,
19089                reviewer: 1,
19090                agent: "alpha".to_owned(),
19091                summary: String::new(),
19092                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
19093                vote: None,
19094                failed: None,
19095                duration_ms: 0,
19096            }],
19097            e2e: vec![CommandOutcome {
19098                command: "cargo test".to_owned(),
19099                code: Some(0),
19100                output_tail: String::new(),
19101                duration_ms: 0,
19102                resource_blocked: false,
19103            }],
19104            verify_retried: false,
19105            e2e_deferred: false,
19106            e2e_defer_reason: None,
19107            fix: Some(FixRecord {
19108                agent: "alpha".to_owned(),
19109                addressed: Vec::new(),
19110                rejected: vec![crate::verdict::Rejection {
19111                    id: "R1-1-1".to_owned(),
19112                    why: "not reachable from any caller".to_owned(),
19113                }],
19114                notes: String::new(),
19115                committed: true,
19116                failed: None,
19117                duration_ms: 0,
19118                continuation: None,
19119            }),
19120            blocking: 0,
19121            answered: 1,
19122            expected: 1,
19123            clean: false,
19124            progressed: true,
19125            vote_split: false,
19126            reconsideration: Vec::new(),
19127            verdict: None,
19128        };
19129        let state = state_with_round(round);
19130        let body = pr_message(&state, 'A').body;
19131
19132        assert!(body.contains("add retries"), "the task must still be there");
19133        assert!(body.contains("R2-1-1"), "{body}");
19134        assert!(!body.contains("unused import"), "{body}");
19135        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
19136        assert!(
19137            !body.contains("not reachable from any caller"),
19138            "the reason it was declined: {body}"
19139        );
19140    }
19141
19142    #[test]
19143    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
19144        let round = ReviewRound {
19145            round: 1,
19146            head: "deadbee".to_owned(),
19147            verified_head: None,
19148            verified_at: None,
19149            reviews: vec![ReviewRecord {
19150                attempts: 0,
19151                reviewer: 1,
19152                agent: "alpha".to_owned(),
19153                summary: String::new(),
19154                findings: Vec::new(),
19155                vote: None,
19156                failed: None,
19157                duration_ms: 0,
19158            }],
19159            e2e: Vec::new(),
19160            verify_retried: false,
19161            e2e_deferred: false,
19162            e2e_defer_reason: None,
19163            fix: None,
19164            blocking: 0,
19165            answered: 1,
19166            expected: 1,
19167            clean: true,
19168            progressed: false,
19169            vote_split: false,
19170            reconsideration: Vec::new(),
19171            verdict: None,
19172        };
19173        let state = state_with_round(round);
19174        let body = pr_message(&state, 'A').body;
19175        assert!(!body.contains("Open review findings"), "{body}");
19176        assert!(!body.contains("Declined"), "{body}");
19177    }
19178
19179    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
19180        let mut state = RunState::new(
19181            PathBuf::from("/repo"),
19182            "main".to_owned(),
19183            "abc1234".to_owned(),
19184            instruction.to_owned(),
19185            Config::default(),
19186        );
19187        state.candidates.push(Candidate {
19188            index: 0,
19189            label: 'A',
19190            agent: "alpha".to_owned(),
19191            branch: "magi/x/A".to_owned(),
19192            worktree: PathBuf::from("/wt"),
19193            summary: summary.to_owned(),
19194            stat: String::new(),
19195            files: 1,
19196            commits: 1,
19197            empty: false,
19198            failed: None,
19199            verified_noop: None,
19200            folded: false,
19201            duration_ms: 0,
19202        });
19203        state
19204    }
19205
19206    fn review_state(subjects: &[&str]) -> RunState {
19207        let mut state = state_with_summary(
19208            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
19209            "",
19210        );
19211        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19212        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
19213        state
19214    }
19215
19216    /// A branch rebased onto a main that moved past the recorded
19217    /// `base_commit` is titled from its own first commit, never main's.
19218    #[tokio::test]
19219    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
19220        ask_test_home();
19221        let tmp = tempfile::tempdir().unwrap();
19222        let repo = tmp.path().join("repo");
19223        std::fs::create_dir_all(&repo).unwrap();
19224        init_repo(&repo);
19225        let origin = tmp.path().join("origin.git");
19226        let g = |dir: &Path, args: &[&str]| {
19227            let out = std::process::Command::new("git")
19228                .args(args)
19229                .current_dir(dir)
19230                .quiet()
19231                .output()
19232                .expect("spawn git");
19233            assert!(
19234                out.status.success(),
19235                "git {args:?}: {}",
19236                String::from_utf8_lossy(&out.stderr)
19237            );
19238        };
19239        g(
19240            tmp.path(),
19241            &[
19242                "clone",
19243                "--bare",
19244                "-q",
19245                repo.to_str().unwrap(),
19246                origin.to_str().unwrap(),
19247            ],
19248        );
19249        g(
19250            &repo,
19251            &["remote", "add", "origin", origin.to_str().unwrap()],
19252        );
19253        let c1 = git::rev_parse(&repo, "main").await.unwrap();
19254
19255        // Main moves on; the branch is built on top of the new main.
19256        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
19257        g(&repo, &["add", "-A"]);
19258        g(
19259            &repo,
19260            &["commit", "-q", "-m", "chore(deps): update a crate"],
19261        );
19262        g(&repo, &["push", "-q", "origin", "main"]);
19263        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
19264        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19265        g(&repo, &["add", "-A"]);
19266        g(
19267            &repo,
19268            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
19269        );
19270        g(&repo, &["checkout", "-q", "main"]);
19271
19272        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
19273        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
19274        // Without a readable tracking ref the recorded base's merge base is used.
19275        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
19276        assert_eq!(fallback, c1);
19277
19278        let mut state = review_state(&[
19279            "chore(deps): update a crate",
19280            "fix(daemon): apply a chosen action",
19281        ]);
19282        state.repo = repo.clone();
19283        state.base_branch = "main".to_owned();
19284        state.base_commit = c1;
19285        refresh_reviewed_commits(&mut state, "feat/own").await;
19286        assert_eq!(
19287            state.reviewed_commits,
19288            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
19289        );
19290        assert_eq!(
19291            review_title(&state).as_deref(),
19292            Some("fix(daemon): apply a chosen action")
19293        );
19294        assert_eq!(
19295            leaked_subjects(&state, "feat/own").await,
19296            Some(vec!["chore(deps): update a crate".to_owned()])
19297        );
19298        // A stale tracking ref is still used when the fetch fails, but the
19299        // leak list is withheld.
19300        state.config.merge.remote = "nowhere".to_owned();
19301        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
19302    }
19303
19304    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
19305    /// with `origin/main` one commit behind it, and a summarizing agent that
19306    /// prints whatever `reply` holds.
19307    struct SummaryFixture {
19308        _tmp: tempfile::TempDir,
19309        repo: PathBuf,
19310        reply: PathBuf,
19311        runner: Runner,
19312    }
19313
19314    fn summary_fixture() -> SummaryFixture {
19315        ask_test_home();
19316        let tmp = tempfile::tempdir().unwrap();
19317        let repo = tmp.path().join("repo");
19318        std::fs::create_dir_all(&repo).unwrap();
19319        init_repo(&repo);
19320        let origin = tmp.path().join("origin.git");
19321        let g = |args: &[&str]| {
19322            let out = std::process::Command::new("git")
19323                .args(args)
19324                .current_dir(&repo)
19325                .quiet()
19326                .output()
19327                .expect("spawn git");
19328            assert!(
19329                out.status.success(),
19330                "git {args:?}: {}",
19331                String::from_utf8_lossy(&out.stderr)
19332            );
19333        };
19334        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
19335        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
19336        g(&["push", "-q", "origin", "main"]);
19337        g(&["checkout", "-q", "-b", "feat/own"]);
19338        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19339        g(&["add", "-A"]);
19340        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
19341        let reply = tmp.path().join("reply.txt");
19342        std::fs::write(
19343            &reply,
19344            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
19345        )
19346        .unwrap();
19347        let mut runner = runner_at(RunStatus::Gating);
19348        let mut state = review_state(&["feat(own): add the own file"]);
19349        state.repo = repo.clone();
19350        state.base_branch = "main".to_owned();
19351        state.candidates[0].branch = "feat/own".to_owned();
19352        state.candidates[0].worktree = repo.clone();
19353        let mut author = spec("alpha");
19354        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
19355        author
19356            .env
19357            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
19358        state.config.agents = vec![author];
19359        runner.state = state;
19360        SummaryFixture {
19361            _tmp: tmp,
19362            repo,
19363            reply,
19364            runner,
19365        }
19366    }
19367
19368    async fn posted(f: &mut SummaryFixture) -> PrMessage {
19369        let winner = f.runner.state.candidates[0].clone();
19370        f.runner
19371            .guarded_pr_message(&winner, None, true)
19372            .await
19373            .unwrap()
19374            .expect("an English description never parks")
19375    }
19376
19377    fn clean_round_with_minor_findings() -> ReviewRound {
19378        let mut round = passing_round(2, None);
19379        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
19380        round.reviews = vec![ReviewRecord {
19381            attempts: 0,
19382            reviewer: 1,
19383            agent: "alpha".into(),
19384            summary: "内部レビュー".into(),
19385            findings: vec![
19386                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
19387                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
19388            ],
19389            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
19390            failed: None,
19391            duration_ms: 0,
19392        }];
19393        round
19394    }
19395
19396    #[tokio::test]
19397    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
19398        for review_only in [false, true] {
19399            let mut f = summary_fixture();
19400            if !review_only {
19401                f.runner.state.candidates[0].agent = "alpha".into();
19402            }
19403            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
19404            assert!(f.runner.state.reviews[0].clean);
19405            assert!(f.runner.state.reviews[0].fix.is_none());
19406            assert!(f.runner.state.open_findings().is_empty());
19407            assert_eq!(
19408                crate::pr_outcome::required_finding_ids(&f.runner.state),
19409                ["R2-1-1", "R2-1-2"]
19410            );
19411            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19412            let evidence = crate::pr_outcome::summary_evidence(
19413                &f.runner.state,
19414                crate::pr_outcome::FinalHead::At(&head),
19415            );
19416            assert!(
19417                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
19418            );
19419            // The ordinary reply omits both IDs: no cached description is adopted.
19420            let omitted = posted(&mut f).await;
19421            assert!(omitted.body.contains("No description"));
19422            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
19423            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
19424            assert!(f.runner.state.change_summary.is_none());
19425            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
19426            let described = posted(&mut f).await;
19427            assert!(described.body.contains("improve documentation for callers"));
19428            assert!(described.body.contains("remaining edge case"));
19429            assert!(described.body.contains("## Open review findings"));
19430            assert!(!described.body.contains("未修正"));
19431        }
19432    }
19433
19434    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
19435        let mut f = summary_fixture();
19436        if !review_only {
19437            f.runner.state.candidates[0].agent = "alpha".into();
19438        }
19439        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19440        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
19441        fixed
19442            .fix
19443            .as_mut()
19444            .unwrap()
19445            .rejected
19446            .push(crate::verdict::Rejection {
19447                id: "R1-1-1".into(),
19448                why: "呼び出されないコードのため辞退".into(),
19449            });
19450        let mut clean = clean_round_with_minor_findings();
19451        clean.verified_head = Some(head.clone());
19452        f.runner.state.reviews = vec![fixed, clean];
19453        ok_gate(&mut f.runner.state, Some(&head));
19454        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
19455        f.runner.state.config.agents.push(scripted("beta", reply));
19456        f.runner.state.config.roles.language_judge =
19457            Some(crate::config::AgentChoice::One("beta".into()));
19458        f
19459    }
19460
19461    #[tokio::test]
19462    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
19463        for review_only in [false, true] {
19464            let mut f = rewrite_fixture_with_findings(review_only,
19465                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
19466            let m = posted(&mut f).await;
19467            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19468            let expected =
19469                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
19470            assert!(m.body.contains(&expected));
19471            assert!(m.body.contains("## Review rounds"));
19472            assert!(
19473                m.body
19474                    .contains("Gate (`verify.gate`): passed on the final commit")
19475            );
19476            assert!(
19477                m.body
19478                    .contains("Tests (`verify.e2e`): passed on the final commit")
19479            );
19480            let c = f.runner.state.change_summary.as_ref().unwrap();
19481            assert!(
19482                m.body
19483                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
19484            );
19485            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19486            assert!(crate::pr_outcome::covers_required_findings(
19487                &f.runner.state,
19488                &m.body
19489            ));
19490            assert!(m.body.contains("documentation remains incomplete"));
19491            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
19492            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
19493            assert!(f.runner.state.github_text.is_none());
19494        }
19495    }
19496
19497    #[tokio::test]
19498    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
19499        for review_only in [false, true] {
19500            for reply in [
19501                "TITLE: fix: public description\nA short English description.",
19502                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
19503            ] {
19504                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
19505                let winner = f.runner.state.candidates[0].clone();
19506                let posted = f
19507                    .runner
19508                    .guarded_pr_message(&winner, None, true)
19509                    .await
19510                    .unwrap();
19511                assert!(posted.is_none());
19512                assert!(f.runner.state.parked);
19513                let withheld = f.runner.state.github_text.as_ref().unwrap();
19514                assert!(
19515                    withheld.rewrite.is_none(),
19516                    "an incomplete rewrite must not be selectable"
19517                );
19518                let q = ask::Questions::open()
19519                    .get(withheld.question.as_ref().unwrap())
19520                    .unwrap();
19521                assert!(
19522                    !q.choices
19523                        .contains(&crate::github_text::USE_REWRITE.to_owned())
19524                );
19525                assert!(
19526                    q.detail.contains("R1-1-1")
19527                        && q.detail.contains("R2-1-1")
19528                        && q.detail.contains("R2-1-2")
19529                );
19530            }
19531        }
19532    }
19533
19534    #[tokio::test]
19535    async fn japanese_reports_stay_local_in_both_public_run_modes() {
19536        for review_only in [false, true] {
19537            let mut f = summary_fixture();
19538            if !review_only {
19539                f.runner.state.candidates[0].agent = "alpha".into();
19540            }
19541            f.runner.state.instruction = "内部タスクの説明".into();
19542            f.runner.state.candidates[0].summary = "内部の候補報告".into();
19543            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19544            f.runner.state.reviews = vec![
19545                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
19546                round_with_notes(2, Some("二回目の内部修正報告です。")),
19547                passing_round(3, Some(&head)),
19548            ];
19549            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
19550            ok_gate(&mut f.runner.state, Some(&head));
19551            let captured = f.repo.join("public-prompt.txt");
19552            let author = &mut f.runner.state.config.agents[0];
19553            author.command = vec![
19554                "sh".into(),
19555                "-c".into(),
19556                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19557            ];
19558            author.env.insert(
19559                "PROMPT_FILE".into(),
19560                captured.to_string_lossy().into_owned(),
19561            );
19562            let m = posted(&mut f).await;
19563            assert!(m.body.contains("Adds `own.txt`"));
19564            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
19565            assert!(m.body.contains("R1-1-1"));
19566            assert!(
19567                m.body
19568                    .contains("Gate (`verify.gate`): passed on the final commit")
19569            );
19570            assert!(
19571                m.body
19572                    .contains("Tests (`verify.e2e`): passed on the final commit")
19573            );
19574            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19575            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
19576            let prompt = std::fs::read_to_string(captured).unwrap();
19577            assert!(prompt.contains(&format!(
19578                "git diff {} {head}",
19579                f.runner.state.change_summary.as_ref().unwrap().base
19580            )));
19581            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
19582            assert_eq!(
19583                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
19584                "二回目の内部修正報告です。"
19585            );
19586        }
19587    }
19588
19589    #[tokio::test]
19590    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
19591        for review_only in [false, true] {
19592            let mut f = summary_fixture();
19593            if !review_only {
19594                f.runner.state.candidates[0].agent = "alpha".into();
19595            }
19596            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
19597            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
19598            std::fs::write(&f.reply, "no usable description").unwrap();
19599            let failed = posted(&mut f).await;
19600            assert!(
19601                failed
19602                    .body
19603                    .contains("No description of what this change does could be generated")
19604            );
19605            assert!(failed.body.is_ascii());
19606            assert!(f.runner.state.github_text.is_none());
19607            assert!(f.runner.state.change_summary.is_none());
19608            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19609            let evidence = crate::pr_outcome::summary_evidence(
19610                &f.runner.state,
19611                crate::pr_outcome::FinalHead::At(&head),
19612            );
19613            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
19614            assert!(
19615                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
19616                    .exists()
19617            );
19618            f.runner.state =
19619                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
19620            std::fs::write(
19621                &f.reply,
19622                "TITLE: feat(own): add the own file\nAdds the committed own file.",
19623            )
19624            .unwrap();
19625            let retried = posted(&mut f).await;
19626            assert!(retried.body.contains("Adds the committed own file."));
19627            assert_eq!(
19628                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
19629                "日本語の修正報告"
19630            );
19631        }
19632    }
19633
19634    #[tokio::test]
19635    async fn same_head_verification_and_findings_invalidate_public_prose() {
19636        let mut f = summary_fixture();
19637        posted(&mut f).await;
19638        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19639        let original = f
19640            .runner
19641            .state
19642            .change_summary
19643            .as_ref()
19644            .unwrap()
19645            .evidence
19646            .clone();
19647        ok_gate(&mut f.runner.state, Some(&head));
19648        std::fs::write(
19649            &f.reply,
19650            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
19651        )
19652        .unwrap();
19653        let verified = posted(&mut f).await;
19654        assert!(verified.body.contains("now verified"));
19655        assert_ne!(
19656            f.runner.state.change_summary.as_ref().unwrap().evidence,
19657            original
19658        );
19659        let mut round = round_with_notes(1, Some("内部報告"));
19660        round.reviews.push(ReviewRecord {
19661            attempts: 0,
19662            reviewer: 1,
19663            agent: "alpha".into(),
19664            summary: "内部レビュー".into(),
19665            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
19666            vote: None,
19667            failed: None,
19668            duration_ms: 0,
19669        });
19670        round
19671            .fix
19672            .as_mut()
19673            .unwrap()
19674            .rejected
19675            .push(crate::verdict::Rejection {
19676                id: "R1-1-1".into(),
19677                why: "呼び出し元から到達できない".into(),
19678            });
19679        f.runner.state.reviews = vec![round];
19680        // A description omitting the required ID is unusable, even with a title.
19681        let omitted = posted(&mut f).await;
19682        assert!(omitted.body.contains("No description"));
19683        assert!(omitted.body.contains("R1-1-1"));
19684        assert!(!omitted.body.contains("到達"));
19685        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
19686        let described = posted(&mut f).await;
19687        assert!(described.body.contains("unreachable from callers"));
19688        assert!(described.body.contains("R1-1-1"));
19689        assert!(!described.body.contains("到達"));
19690        let evidence = f
19691            .runner
19692            .state
19693            .change_summary
19694            .as_ref()
19695            .unwrap()
19696            .evidence
19697            .clone();
19698        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
19699        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
19700        assert!(posted(&mut f).await.body.contains("rationale updated"));
19701        assert_ne!(
19702            f.runner.state.change_summary.as_ref().unwrap().evidence,
19703            evidence
19704        );
19705        // Old serialized summaries have no evidence fingerprint and must regenerate.
19706        f.runner
19707            .state
19708            .change_summary
19709            .as_mut()
19710            .unwrap()
19711            .evidence
19712            .clear();
19713        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
19714        assert!(posted(&mut f).await.body.contains("current specification"));
19715    }
19716
19717    #[tokio::test]
19718    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
19719        let mut f = summary_fixture();
19720        let m = posted(&mut f).await;
19721        assert_eq!(m.title, "feat(own): add the own file");
19722        assert!(
19723            m.body
19724                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
19725            "{}",
19726            m.body
19727        );
19728        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
19729        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
19730        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19731        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19732        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
19733        let kept = f.runner.state.change_summary.as_ref().unwrap();
19734        assert_eq!(
19735            (kept.head.as_str(), kept.base.as_str()),
19736            (head.as_str(), base.as_str())
19737        );
19738        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
19739    }
19740
19741    #[tokio::test]
19742    async fn review_only_summary_receives_the_configured_github_language() {
19743        let mut f = summary_fixture();
19744        f.runner.state.config.graph.github_language = "ja".into();
19745        let captured = f.repo.join("summary-prompt.txt");
19746        let author = &mut f.runner.state.config.agents[0];
19747        author.command = vec![
19748            "sh".into(),
19749            "-c".into(),
19750            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19751        ];
19752        author.env.insert(
19753            "PROMPT_FILE".into(),
19754            captured.to_string_lossy().into_owned(),
19755        );
19756        std::fs::write(
19757            &f.reply,
19758            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
19759        )
19760        .unwrap();
19761        let m = posted(&mut f).await;
19762        let prompt = std::fs::read_to_string(captured).unwrap();
19763        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
19764        assert!(!prompt.contains("in English"));
19765        assert_eq!(m.title, "feat(own): ファイルを追加");
19766        assert!(m.body.contains("own.txt を追加します。"));
19767    }
19768
19769    #[tokio::test]
19770    async fn review_fixes_come_after_the_whole_change_summary() {
19771        let mut f = summary_fixture();
19772        f.runner.state.reviews = vec![
19773            round_with_notes(1, Some("handled the PATH case")),
19774            round_with_notes(2, None),
19775        ];
19776        let m = posted(&mut f).await;
19777        let summary = m.body.find("## Summary").expect("summary");
19778        let fixes = m.body.find("## Review rounds").expect("review rounds");
19779        assert!(summary < fixes, "{}", m.body);
19780        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
19781        assert!(m.body.contains("## Final verification"), "{}", m.body);
19782    }
19783
19784    #[tokio::test]
19785    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
19786        let mut f = summary_fixture();
19787        let first = posted(&mut f).await;
19788        let first_head = f.runner.state.change_summary.clone().unwrap().head;
19789        // The same head reuses the record: a different reply is not read.
19790        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
19791        let again = posted(&mut f).await;
19792        assert_eq!((again.title, again.body), (first.title, first.body));
19793
19794        let g = |args: &[&str]| {
19795            let out = std::process::Command::new("git")
19796                .args(args)
19797                .current_dir(&f.repo)
19798                .quiet()
19799                .output()
19800                .expect("spawn git");
19801            assert!(out.status.success(), "git {args:?}");
19802        };
19803        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
19804        g(&["add", "-A"]);
19805        g(&["commit", "-q", "-m", "fix(own): handle more"]);
19806        let second = posted(&mut f).await;
19807        let kept = f.runner.state.change_summary.clone().unwrap();
19808        assert_ne!(kept.head, first_head);
19809        assert!(second.body.contains("Not read yet."), "{}", second.body);
19810        assert!(!second.body.contains("so the first summary is read"));
19811        assert_eq!(second.title, "feat(own): other");
19812    }
19813
19814    #[tokio::test]
19815    async fn an_unavailable_summary_is_stated_and_never_invented() {
19816        let mut f = summary_fixture();
19817        std::fs::write(&f.reply, "no title line here").unwrap();
19818        let m = posted(&mut f).await;
19819        assert!(
19820            m.body
19821                .contains("No description of what this change does could be generated"),
19822            "{}",
19823            m.body
19824        );
19825        assert!(m.body.contains("internal reports remain in local records"));
19826        assert!(f.runner.state.change_summary.is_none());
19827        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
19828        // A preview asks nobody and claims nothing.
19829        let winner = f.runner.state.candidates[0].clone();
19830        let preview = f
19831            .runner
19832            .guarded_pr_message(&winner, None, false)
19833            .await
19834            .unwrap()
19835            .unwrap();
19836        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
19837    }
19838
19839    #[test]
19840    fn pr_message_review_single_commit_uses_its_subject() {
19841        let state = review_state(&["feat(nats): per-role user"]);
19842        let m = pr_message(&state, 'A');
19843        assert_eq!(m.title, "feat(nats): per-role user");
19844        assert!(!m.body.contains("Review the work already"), "{}", m.body);
19845        assert!(m.body.contains("## Commits under review"), "{}", m.body);
19846    }
19847
19848    #[test]
19849    fn pr_message_review_multi_commit_takes_the_oldest() {
19850        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
19851        let m = pr_message(&state, 'A');
19852        assert_eq!(m.title, "feat: the change");
19853        for s in ["feat: the change", "fix: typo", "fix: again"] {
19854            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
19855        }
19856    }
19857
19858    #[test]
19859    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
19860        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
19861            let mut state = review_state(&[first, "fix: later fixup"]);
19862            state.candidates[0].branch = "機能/ログイン".to_owned();
19863            let m = pr_message(&state, 'A');
19864            assert!(
19865                m.title.starts_with("chore: land candidate A of run"),
19866                "{}",
19867                m.title
19868            );
19869        }
19870    }
19871
19872    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
19873        BranchFacts {
19874            commits: commits
19875                .iter()
19876                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
19877                .collect(),
19878            stat: stat.to_owned(),
19879        }
19880    }
19881
19882    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
19883        let mut r = review_round(true, 0, 1, 1, true, true);
19884        r.round = round;
19885        r.fix = notes.map(|n| FixRecord {
19886            agent: "fixer".to_owned(),
19887            addressed: Vec::new(),
19888            rejected: Vec::new(),
19889            notes: n.to_owned(),
19890            committed: true,
19891            failed: None,
19892            duration_ms: 0,
19893            continuation: None,
19894        });
19895        r
19896    }
19897
19898    #[test]
19899    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
19900        let state = review_state(&["ignored"]);
19901        let f = facts(
19902            &[
19903                (
19904                    "fix(login): resolve PATH on macOS",
19905                    "Login shells skip rc files.",
19906                ),
19907                ("fix: address review", ""),
19908            ],
19909            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
19910        );
19911        let m = pr_message_with(&state, 'A', Some(&f));
19912        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
19913        assert!(
19914            m.body
19915                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
19916            "{}",
19917            m.body
19918        );
19919        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
19920        assert!(m.body.contains("## Diff stat"), "{}", m.body);
19921        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
19922        for banned in [
19923            "Review the work already",
19924            "no task statement",
19925            "Original task",
19926        ] {
19927            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
19928        }
19929        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
19930    }
19931
19932    #[test]
19933    fn pr_message_review_truncates_a_huge_first_commit_body() {
19934        let state = review_state(&["ignored"]);
19935        let f = facts(
19936            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
19937            "s",
19938        );
19939        let m = pr_message_with(&state, 'A', Some(&f));
19940        assert!(m.body.len() < 30_000, "{}", m.body.len());
19941        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
19942        assert!(
19943            m.body.contains("- ... 1 more commit(s)"),
19944            "{}",
19945            m.body.len()
19946        );
19947        assert!(m.body.ends_with("magi:candidate-a\n"));
19948    }
19949
19950    #[test]
19951    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
19952        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
19953        assert_eq!(m.title, "feat: x");
19954        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
19955        assert!(!m.body.contains("Diff stat"), "{}", m.body);
19956        assert!(!m.body.contains("no task statement"), "{}", m.body);
19957    }
19958
19959    #[test]
19960    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
19961        let mut state = review_state(&["日本語の件名"]);
19962        state.candidates[0].branch = "fix/macos-login-path".to_owned();
19963        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
19964        state.candidates[0].branch = "機能/ログイン".to_owned();
19965        assert!(
19966            pr_message(&state, 'A')
19967                .title
19968                .starts_with("chore: land candidate A")
19969        );
19970    }
19971
19972    #[test]
19973    fn pr_message_review_fixes_survive_a_clean_final_round() {
19974        let mut state = review_state(&["feat: x"]);
19975        state.reviews = vec![
19976            round_with_notes(1, Some("handled the PATH case")),
19977            round_with_notes(2, None),
19978        ];
19979        let body = pr_message(&state, 'A').body;
19980        assert!(
19981            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
19982            "{body}"
19983        );
19984        assert!(body.contains("### Round 1"), "{body}");
19985
19986        state.reviews = vec![
19987            round_with_notes(1, Some("first fix")),
19988            round_with_notes(2, Some("")),
19989            round_with_notes(3, Some("second fix")),
19990            round_with_notes(4, None),
19991        ];
19992        let body = pr_message(&state, 'A').body;
19993        assert!(
19994            body.contains("### Round 1") && !body.contains("first fix"),
19995            "{body}"
19996        );
19997        assert!(
19998            body.contains("### Round 3") && !body.contains("second fix"),
19999            "{body}"
20000        );
20001        assert!(body.contains("### Round 2"), "{body}");
20002    }
20003
20004    const FINAL: &str = "abcdef1234567890";
20005
20006    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
20007        let mut r = round_with_notes(round, None);
20008        r.verified_head = verified.map(str::to_owned);
20009        r
20010    }
20011
20012    fn ok_gate(state: &mut RunState, head: Option<&str>) {
20013        state.gate = vec![CommandOutcome {
20014            command: "gate".to_owned(),
20015            code: Some(0),
20016            output_tail: String::new(),
20017            duration_ms: 0,
20018            resource_blocked: false,
20019        }];
20020        state.gate_ran = true;
20021        state.gate_head = head.map(str::to_owned);
20022    }
20023
20024    fn body_at(state: &RunState) -> String {
20025        pr_message_raw(
20026            state,
20027            'A',
20028            None,
20029            crate::pr_outcome::FinalHead::At(FINAL),
20030            ChangeNote::NotAsked,
20031        )
20032        .body
20033    }
20034
20035    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
20036
20037    #[test]
20038    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
20039        let mut state = review_state(&["feat: x"]);
20040        state.reviews = vec![
20041            round_with_notes(1, Some(SANDBOXED)),
20042            passing_round(2, Some(FINAL)),
20043        ];
20044        ok_gate(&mut state, Some(FINAL));
20045        let body = body_at(&state);
20046        assert!(!body.contains("Handled the PATH case."), "{body}");
20047        assert!(!body.contains("could not commit"), "{body}");
20048        assert!(!body.contains("could not run"), "{body}");
20049        assert!(
20050            body.contains("The fix is committed on the branch"),
20051            "{body}"
20052        );
20053        assert!(!body.contains("sandbox"), "{body}");
20054        assert!(
20055            body.contains("Tests (`verify.e2e`): passed on the final commit"),
20056            "{body}"
20057        );
20058        assert!(
20059            body.contains("Gate (`verify.gate`): passed on the final commit"),
20060            "{body}"
20061        );
20062        // The raw report is untouched in the record.
20063        assert!(
20064            state.reviews[0]
20065                .fix
20066                .as_ref()
20067                .unwrap()
20068                .notes
20069                .contains("could not commit")
20070        );
20071    }
20072
20073    #[test]
20074    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
20075        let mut state = review_state(&["feat: x"]);
20076        let mut r1 = round_with_notes(1, Some(SANDBOXED));
20077        r1.fix.as_mut().unwrap().committed = false;
20078        let mut r2 = passing_round(2, Some(FINAL));
20079        r2.e2e[0].code = Some(1);
20080        state.reviews = vec![r1, r2];
20081        let body = body_at(&state);
20082        assert!(!body.contains("could not commit"), "{body}");
20083        assert!(!body.contains("could not run"), "{body}");
20084        assert!(body.contains("This round produced no commit"), "{body}");
20085        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
20086        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
20087        assert!(!body.contains("passed"), "{body}");
20088    }
20089
20090    #[test]
20091    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
20092        let mut state = review_state(&["feat: x"]);
20093        state.reviews = vec![
20094            round_with_notes(1, Some(SANDBOXED)),
20095            passing_round(2, Some("0123456789abcdef")),
20096        ];
20097        ok_gate(&mut state, Some("0123456789abcdef"));
20098        let body = body_at(&state);
20099        assert!(!body.contains("HTTP tests could not run"), "{body}");
20100        // The commit claim is still settled by the structured record.
20101        assert!(!body.contains("could not commit"), "{body}");
20102        assert!(
20103            body.contains("passed on an earlier commit (`0123456`)"),
20104            "{body}"
20105        );
20106        assert!(!body.contains("passed on the final commit"), "{body}");
20107
20108        // A gate recorded before `gate_head` existed is "not recorded".
20109        ok_gate(&mut state, None);
20110        let body = body_at(&state);
20111        assert!(
20112            body.contains("passed on a commit that was not recorded"),
20113            "{body}"
20114        );
20115    }
20116
20117    #[test]
20118    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
20119        let state = state_with_summary(
20120            "add retries to the client",
20121            "TITLE: feat: retries\n\nDid it.",
20122        );
20123        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
20124        assert_eq!(m.title, "feat: retries");
20125        assert!(
20126            m.body.contains("<summary>Original task</summary>"),
20127            "{}",
20128            m.body
20129        );
20130        assert!(!m.body.contains("Commits under review"), "{}", m.body);
20131        assert!(
20132            m.body
20133                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
20134            "{}",
20135            m.body
20136        );
20137    }
20138
20139    #[test]
20140    fn pr_message_review_bounds_a_long_english_subject() {
20141        let long = format!("feat: {}", "word ".repeat(100));
20142        let m = pr_message(&review_state(&[&long]), 'A');
20143        assert!(m.title.starts_with("feat: word"), "{}", m.title);
20144        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20145    }
20146
20147    #[test]
20148    fn pr_message_implementation_run_is_unchanged_by_review_support() {
20149        let state = state_with_summary("add retries\n\ndetails", "- did some things");
20150        let m = pr_message(&state, 'A');
20151        assert_eq!(m.title, "add retries");
20152        assert!(m.body.contains("<summary>Original task</summary>"));
20153        assert!(!m.body.contains("Commits under review"));
20154        assert_eq!(landing_subject_source(&state), state.instruction);
20155    }
20156
20157    #[test]
20158    fn review_run_squash_subject_is_the_change_not_the_prompt() {
20159        let state = review_state(&["feat: the change", "fix: typo"]);
20160        let source = landing_subject_source(&state);
20161        assert_eq!(land::merge_subject("", &source), "feat: the change");
20162        assert_eq!(
20163            land::merge_subject("magi: candidate A (uncommitted work)", &source),
20164            "feat: the change"
20165        );
20166        // An operator's rename still wins.
20167        assert_eq!(
20168            land::merge_subject("feat: renamed by hand", &source),
20169            "feat: renamed by hand"
20170        );
20171        let blank = review_state(&["日本語"]);
20172        assert!(
20173            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
20174        );
20175    }
20176
20177    #[test]
20178    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
20179        let state = review_state(&["feat: the change"]);
20180        let source = landing_subject_source(&state);
20181        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
20182        assert_eq!(
20183            land::merge_subject(landing_title(&state, old), &source),
20184            "feat: the change"
20185        );
20186        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
20187        let task = state_with_summary("add retries", "");
20188        assert_eq!(landing_title(&task, old), old);
20189    }
20190
20191    #[test]
20192    fn pr_message_describes_the_change_not_the_task() {
20193        let state = state_with_summary(
20194            "今回やってほしいこと: results projector を直す",
20195            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
20196        );
20197        let m = pr_message(&state, 'A');
20198        assert_eq!(m.title, "fix(web): batch the runs list reads");
20199        assert!(
20200            m.body.starts_with("## Summary\n\n- reads run.json once"),
20201            "{}",
20202            m.body
20203        );
20204        assert!(!m.body.contains("TITLE:"), "{}", m.body);
20205        let task_at = m.body.find("今回やってほしいこと").unwrap();
20206        let details_at = m.body.find("<details>").unwrap();
20207        assert!(
20208            details_at < task_at,
20209            "the task lives inside <details>: {}",
20210            m.body
20211        );
20212        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
20213        assert!(m.body.contains("magi:candidate-a"));
20214    }
20215
20216    #[test]
20217    fn pr_message_falls_back_to_the_task_without_a_title_line() {
20218        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
20219        let m = pr_message(&state, 'A');
20220        assert_eq!(m.title, "add retries");
20221        assert!(
20222            m.body.contains("## Summary\n\n- did some things"),
20223            "{}",
20224            m.body
20225        );
20226
20227        let none = RunState::new(
20228            PathBuf::from("/repo"),
20229            "main".to_owned(),
20230            "abc1234".to_owned(),
20231            "add retries".to_owned(),
20232            Config::default(),
20233        );
20234        let m = pr_message(&none, 'A');
20235        assert_eq!(m.title, "add retries");
20236        assert!(!m.body.contains("## Summary"), "{}", m.body);
20237    }
20238
20239    #[test]
20240    fn pr_message_refuses_the_candidate_commit_subject() {
20241        for bad in [
20242            "TITLE: magi: candidate A (uncommitted work)",
20243            "TITLE: chore: stuff (uncommitted work)",
20244            "TITLE:   ",
20245        ] {
20246            let state = state_with_summary("add retries", bad);
20247            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
20248        }
20249    }
20250
20251    #[test]
20252    fn pr_message_bounds_a_very_long_task_and_title() {
20253        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
20254        let state = state_with_summary(&long, "- nothing");
20255        let m = pr_message(&state, 'A');
20256        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20257        assert!(!m.title.contains('\n'));
20258
20259        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
20260        let m = pr_message(&state, 'A');
20261        assert!(m.title.starts_with("feat: "));
20262        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20263        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
20264    }
20265
20266    fn long_title_of(instruction: &str) -> String {
20267        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
20268    }
20269
20270    #[test]
20271    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
20272        let first = "Make the landing path keep a readable title for long tasks";
20273        let line = format!(
20274            "{first}. {}",
20275            "And then keep going with more words ".repeat(20)
20276        );
20277        let t = long_title_of(&line);
20278        assert_eq!(t, first);
20279        assert!(!t.starts_with("chore: land"));
20280    }
20281
20282    #[test]
20283    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
20284        let line = "word ".repeat(200);
20285        let t = long_title_of(&line);
20286        assert!(t.ends_with("word..."), "{t}");
20287        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
20288    }
20289
20290    #[test]
20291    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
20292        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
20293            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
20294        }
20295    }
20296
20297    #[test]
20298    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
20299        let mut state = state_with_summary("再試行を追加する", "- nothing");
20300        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
20301        state.config.graph.github_language = "ja".into();
20302        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
20303    }
20304
20305    #[test]
20306    fn pr_message_title_limit_is_exact() {
20307        let at = "a".repeat(PR_TITLE_MAX);
20308        assert_eq!(long_title_of(&at), at);
20309        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
20310        assert!(over.ends_with("..."), "{over}");
20311        assert_eq!(over.chars().count(), PR_TITLE_MAX);
20312    }
20313
20314    #[test]
20315    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
20316        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
20317        let t = long_title_of(&line);
20318        assert!(t.ends_with("..."), "{t}");
20319        assert!(t.is_ascii(), "{t}");
20320    }
20321
20322    #[test]
20323    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
20324        let line = format!(
20325            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
20326            "plus more filler words ".repeat(20)
20327        );
20328        let t = long_title_of(&line);
20329        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
20330    }
20331
20332    #[test]
20333    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
20334        let line = format!("\n\n# {}", "title words ".repeat(40));
20335        let state = state_with_summary(&line, "- nothing");
20336        let m = pr_message(&state, 'A');
20337        assert!(m.title.starts_with("title words"), "{}", m.title);
20338        assert_eq!(
20339            land::merge_subject(&m.title, &landing_subject_source(&state)),
20340            m.title
20341        );
20342        // An operator's rename wins untouched.
20343        assert_eq!(
20344            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
20345            "feat: renamed by hand"
20346        );
20347    }
20348
20349    #[tokio::test]
20350    async fn github_text_rewrite_is_bounded_and_falls_back() {
20351        crate::run::pin_test_home();
20352        for (reply, accepted) in [
20353            (
20354                "TITLE: fix: retries\nAdd retries for failed requests.",
20355                true,
20356            ),
20357            (
20358                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
20359                false,
20360            ),
20361            (
20362                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
20363                true,
20364            ),
20365            (
20366                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
20367                true,
20368            ),
20369            (
20370                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
20371                true,
20372            ),
20373            (
20374                "TITLE: fix: retries\n日本語の説明です。 token=secret",
20375                false,
20376            ),
20377            ("TITLE: fix: retries\n", false),
20378        ] {
20379            let dir = tempfile::tempdir().unwrap();
20380            let mut runner = runner_at(RunStatus::Gating);
20381            runner.state = state_with_summary(
20382                "add retries",
20383                "TITLE: fix: retries\n日本語の説明を書きます。",
20384            );
20385            runner.state.repo = dir.path().to_owned();
20386            runner.state.candidates[0].worktree = dir.path().to_owned();
20387            let mut author = spec("alpha");
20388            author.command = vec![
20389                "sh".into(),
20390                "-c".into(),
20391                "printf '%s' \"$REWRITE_REPLY\"".into(),
20392            ];
20393            author.env.insert("REWRITE_REPLY".into(), reply.into());
20394            runner.state.config.agents = vec![author];
20395            seed_public_summary(&mut runner);
20396            let winner = runner.state.candidates[0].clone();
20397            let first = runner
20398                .guarded_pr_message(&winner, None, true)
20399                .await
20400                .unwrap();
20401            let message = if accepted {
20402                first.expect("a passing rewrite never asks")
20403            } else {
20404                // Withheld: the run parks on a question and posts nothing yet.
20405                assert!(first.is_none());
20406                assert!(runner.state.parked);
20407                let store = ask::Questions::open();
20408                let asked: Vec<_> = store
20409                    .list()
20410                    .into_iter()
20411                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20412                    .collect();
20413                assert_eq!(asked.len(), 1);
20414                assert!(!asked[0].detail.contains("secret"));
20415                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
20416                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
20417                assert_eq!(
20418                    asked[0]
20419                        .choices
20420                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
20421                    rewrite.is_some()
20422                );
20423                // Silence: the question lapses and the neutral text is used,
20424                // without a second rewrite call or a second question.
20425                store
20426                    .update(&asked[0].id, |q| {
20427                        q.abandon("test");
20428                        Ok(())
20429                    })
20430                    .unwrap();
20431                let again = runner
20432                    .guarded_pr_message(&winner, None, true)
20433                    .await
20434                    .unwrap()
20435                    .expect("an abandoned question falls back");
20436                let n = store
20437                    .list()
20438                    .into_iter()
20439                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20440                    .count();
20441                assert_eq!(n, 1, "asked once per rejected text");
20442                again
20443            };
20444            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
20445            assert!(!message.body.contains("secret"));
20446            assert!(!message.body.contains("prefix"));
20447            assert!(!message.title.contains("secret"));
20448            assert_eq!(
20449                message.body.contains("Add retries for failed requests."),
20450                accepted
20451            );
20452            assert_eq!(
20453                runner.state.seats["github-text-rewrite"].turns, 1,
20454                "only one rewrite invocation"
20455            );
20456            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
20457            if accepted {
20458                assert!(runner.state.github_text.is_none());
20459                assert!(!runner.state.parked);
20460                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
20461                assert!(message.body.contains("## Final verification"));
20462                assert!(message.body.contains("Evidence: final diff"));
20463                assert!(
20464                    message
20465                        .body
20466                        .contains(&format!("magi:run/{}", runner.state.id))
20467                );
20468            }
20469            if !accepted {
20470                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20471                assert!(
20472                    message
20473                        .body
20474                        .contains(&format!("magi:run/{}", runner.state.id))
20475                );
20476            }
20477        }
20478    }
20479
20480    fn scripted(id: &str, reply: &str) -> AgentSpec {
20481        let mut a = spec(id);
20482        a.command = vec![
20483            "sh".into(),
20484            "-c".into(),
20485            "printf '%s' \"$REWRITE_REPLY\"".into(),
20486        ];
20487        a.env.insert("REWRITE_REPLY".into(), reply.into());
20488        a
20489    }
20490
20491    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
20492    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
20493
20494    /// Gate tests inject generated public output, not internal reports.
20495    fn seed_public_summary(runner: &mut Runner) {
20496        let repo = &runner.state.repo;
20497        let g = |args: &[&str]| {
20498            let out = std::process::Command::new("git")
20499                .args(args)
20500                .current_dir(repo)
20501                .quiet()
20502                .output()
20503                .unwrap();
20504            assert!(out.status.success(), "git {args:?}");
20505            String::from_utf8_lossy(&out.stdout).trim().to_owned()
20506        };
20507        g(&["init", "-q", "-b", "main"]);
20508        g(&[
20509            "-c",
20510            "user.name=Test",
20511            "-c",
20512            "user.email=test@example.invalid",
20513            "commit",
20514            "-q",
20515            "--allow-empty",
20516            "-m",
20517            "feat: fixture",
20518        ]);
20519        let head = g(&["rev-parse", "HEAD"]);
20520        g(&["update-ref", "refs/remotes/origin/main", &head]);
20521        let evidence = crate::pr_outcome::summary_evidence(
20522            &runner.state,
20523            crate::pr_outcome::FinalHead::At(&head),
20524        );
20525        let generated = &runner.state.candidates[0].summary;
20526        runner.state.change_summary = Some(crate::run::ChangeSummary {
20527            base: head.clone(),
20528            head,
20529            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
20530            title: summary_title(generated).unwrap(),
20531            body: summary_without_title(generated),
20532        });
20533    }
20534
20535    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
20536        let mut runner = runner_at(RunStatus::Gating);
20537        runner.state = state_with_summary("add retries", JA_SUMMARY);
20538        if review_only {
20539            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
20540        }
20541        runner.state.repo = dir.to_owned();
20542        runner.state.candidates[0].worktree = dir.to_owned();
20543        runner.state.config.agents = agents;
20544        seed_public_summary(&mut runner);
20545        runner
20546    }
20547
20548    fn gate_questions(runner: &Runner) -> usize {
20549        ask::Questions::open()
20550            .list()
20551            .into_iter()
20552            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20553            .count()
20554    }
20555
20556    #[tokio::test]
20557    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
20558        crate::run::pin_test_home();
20559        let dir = tempfile::tempdir().unwrap();
20560        let mut runner = rewrite_runner(
20561            dir.path(),
20562            true,
20563            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20564        );
20565        runner.state.config.roles.language_judge =
20566            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
20567        let winner = runner.state.candidates[0].clone();
20568        let msg = runner
20569            .guarded_pr_message(&winner, None, true)
20570            .await
20571            .unwrap()
20572            .expect("an English rewrite posts without asking");
20573        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
20574        assert!(msg.body.contains("<details>kept</details>"));
20575        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
20576        assert_eq!(gate_questions(&runner), 0);
20577        assert!(runner.state.github_text.is_none());
20578        assert!(!runner.state.seats.contains_key("impl-A"));
20579        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20580    }
20581
20582    #[tokio::test]
20583    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
20584        crate::run::pin_test_home();
20585        let dir = tempfile::tempdir().unwrap();
20586        let mut runner = rewrite_runner(
20587            dir.path(),
20588            false,
20589            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20590        );
20591        runner.state.config.roles.language_judge =
20592            Some(crate::config::AgentChoice::One("beta".into()));
20593        runner.state.seats.insert(
20594            "impl-A".into(),
20595            SeatState::new("impl-A", "alpha", runner.state.seed),
20596        );
20597        let winner = runner.state.candidates[0].clone();
20598        let msg = runner
20599            .guarded_pr_message(&winner, None, true)
20600            .await
20601            .unwrap()
20602            .expect("rewritten");
20603        assert!(msg.body.contains("<details>kept</details>"));
20604        assert_eq!(runner.state.seats["impl-A"].turns, 0);
20605        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20606        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20607    }
20608
20609    #[tokio::test]
20610    async fn rewrite_reuses_a_resumable_implementer_session() {
20611        crate::run::pin_test_home();
20612        let dir = tempfile::tempdir().unwrap();
20613        let mut runner = rewrite_runner(
20614            dir.path(),
20615            false,
20616            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
20617        );
20618        runner.state.config.roles.language_judge =
20619            Some(crate::config::AgentChoice::One("beta".into()));
20620        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20621        seat.turns = 1;
20622        runner.state.seats.insert("impl-A".into(), seat);
20623        let winner = runner.state.candidates[0].clone();
20624        runner
20625            .guarded_pr_message(&winner, None, true)
20626            .await
20627            .unwrap()
20628            .expect("rewritten");
20629        assert_eq!(runner.state.seats["impl-A"].turns, 2);
20630        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
20631    }
20632
20633    #[tokio::test]
20634    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
20635        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
20636        // session, was out of quota, and nothing else was asked.
20637        crate::run::pin_test_home();
20638        let dir = tempfile::tempdir().unwrap();
20639        let mut exhausted = spec("alpha");
20640        exhausted.command = vec!["false".into()];
20641        let mut runner = rewrite_runner(
20642            dir.path(),
20643            false,
20644            vec![exhausted, scripted("beta", EN_REWRITE)],
20645        );
20646        // The implementer's own agent is in the role too: it is not asked twice.
20647        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20648            "alpha".into(),
20649            "beta".into(),
20650        ]));
20651        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20652        seat.turns = 1;
20653        runner.state.seats.insert("impl-A".into(), seat);
20654        let winner = runner.state.candidates[0].clone();
20655        let msg = runner
20656            .guarded_pr_message(&winner, None, true)
20657            .await
20658            .unwrap()
20659            .expect("the role rewrites after the implementer fails");
20660        assert!(msg.body.contains("`src/net.rs`"));
20661        assert_eq!(
20662            runner.state.seats["impl-A"].turns, 2,
20663            "the implementer was tried once"
20664        );
20665        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20666        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20667    }
20668
20669    #[tokio::test]
20670    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
20671        crate::run::pin_test_home();
20672        let dir = tempfile::tempdir().unwrap();
20673        let mut broken = spec("bad");
20674        broken.command = vec!["false".into()];
20675        let mut runner = rewrite_runner(
20676            dir.path(),
20677            true,
20678            vec![
20679                broken,
20680                scripted("noformat", "Add retries."),
20681                scripted("good", EN_REWRITE),
20682            ],
20683        );
20684        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20685            "bad".into(),
20686            "noformat".into(),
20687            "good".into(),
20688        ]));
20689        let winner = runner.state.candidates[0].clone();
20690        let msg = runner
20691            .guarded_pr_message(&winner, None, true)
20692            .await
20693            .unwrap()
20694            .expect("the third agent rewrites");
20695        assert!(msg.body.contains("`src/net.rs`"));
20696        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
20697        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
20698    }
20699
20700    #[tokio::test]
20701    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
20702        crate::run::pin_test_home();
20703        // (agents, role): still Japanese; no TITLE; nothing resolvable.
20704        for (agents, role, saved) in [
20705            (
20706                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
20707                Some("beta"),
20708                true,
20709            ),
20710            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
20711            (Vec::new(), None, false),
20712        ] {
20713            let dir = tempfile::tempdir().unwrap();
20714            let mut runner = rewrite_runner(dir.path(), true, agents);
20715            runner.state.config.roles.language_judge =
20716                role.map(|r| crate::config::AgentChoice::One(r.into()));
20717            let winner = runner.state.candidates[0].clone();
20718            let first = runner
20719                .guarded_pr_message(&winner, None, true)
20720                .await
20721                .unwrap();
20722            assert!(first.is_none(), "withheld text parks the run");
20723            assert!(runner.state.parked);
20724            assert_eq!(gate_questions(&runner), 1);
20725            let g = runner.state.github_text.as_ref().unwrap();
20726            assert_eq!(g.rewrite.is_some(), saved);
20727            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
20728        }
20729    }
20730
20731    #[tokio::test]
20732    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
20733        crate::run::pin_test_home();
20734        let dir = tempfile::tempdir().unwrap();
20735        let mut runner = runner_at(RunStatus::Gating);
20736        runner.state =
20737            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
20738        runner.state.repo = dir.path().to_owned();
20739        runner.state.candidates[0].worktree = dir.path().to_owned();
20740        let mut author = spec("alpha");
20741        author.command = vec![
20742            "sh".into(),
20743            "-c".into(),
20744            "printf '%s' \"$REWRITE_REPLY\"".into(),
20745        ];
20746        author.env.insert(
20747            "REWRITE_REPLY".into(),
20748            format!(
20749                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
20750                "English sentence. ".repeat(600)
20751            ),
20752        );
20753        runner.state.config.agents = vec![author];
20754        seed_public_summary(&mut runner);
20755        let winner = runner.state.candidates[0].clone();
20756        assert!(
20757            runner
20758                .guarded_pr_message(&winner, None, true)
20759                .await
20760                .unwrap()
20761                .is_none()
20762        );
20763        let record = runner.state.github_text.clone().unwrap();
20764        let rewrite = record.rewrite.clone().unwrap();
20765        assert_eq!(rewrite.failed, ["body-language"]);
20766        assert!(
20767            rewrite
20768                .body
20769                .ends_with(&format!("magi:run/{}", runner.state.id))
20770        );
20771        let store = ask::Questions::open();
20772        let q = store.get(record.question.as_ref().unwrap()).unwrap();
20773        assert!(
20774            q.choices
20775                .contains(&crate::github_text::USE_REWRITE.to_owned())
20776        );
20777        for label in [
20778            "ORIGINAL",
20779            "FALLBACK",
20780            "REWRITE",
20781            "Rules still failed: body-language",
20782            "truncated",
20783        ] {
20784            assert!(q.detail.contains(label), "{label}");
20785        }
20786        let artifact = std::fs::read_to_string(crate::run::artifact_path(
20787            &runner.state,
20788            &crate::github_text::artifact_name(&record.fingerprint),
20789        ))
20790        .unwrap();
20791        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
20792        // Restart from serialized state, then retry after a bad owner title.
20793        runner.state =
20794            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
20795        store
20796            .update(&q.id, |q| {
20797                q.say("まだ日本語")?;
20798                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
20799            })
20800            .unwrap();
20801        assert!(
20802            runner
20803                .guarded_pr_message(&winner, None, true)
20804                .await
20805                .unwrap()
20806                .is_none()
20807        );
20808        let retry = runner.state.github_text.as_ref().unwrap();
20809        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
20810        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
20811        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
20812        assert!(
20813            q.posting_comparison
20814                .as_ref()
20815                .unwrap()
20816                .matches_detail(&q.detail)
20817        );
20818        store
20819            .update(&q.id, |q| {
20820                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20821            })
20822            .unwrap();
20823        let message = runner
20824            .guarded_pr_message(&winner, None, true)
20825            .await
20826            .unwrap()
20827            .unwrap();
20828        assert_eq!(message.title, rewrite.title);
20829        assert_eq!(message.body, rewrite.body);
20830        assert_eq!(
20831            runner.state.github_text.as_ref().unwrap().choice,
20832            Some(GithubTextChoice::Rewrite)
20833        );
20834        let again = runner
20835            .guarded_pr_message(&winner, None, true)
20836            .await
20837            .unwrap()
20838            .unwrap();
20839        assert_eq!(again.title, message.title);
20840        assert_eq!(again.body, message.body);
20841        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20842        assert!(
20843            !runner
20844                .state
20845                .events
20846                .iter()
20847                .any(|e| e.message.contains(&rewrite.title)
20848                    || e.message.contains("English sentence."))
20849        );
20850    }
20851
20852    #[tokio::test]
20853    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
20854        crate::run::pin_test_home();
20855        let dir = tempfile::tempdir().unwrap();
20856        let mut runner = runner_at(RunStatus::Gating);
20857        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
20858        runner.state.repo = dir.path().to_owned();
20859        runner.state.candidates[0].worktree = dir.path().to_owned();
20860        seed_public_summary(&mut runner);
20861        let winner = runner.state.candidates[0].clone();
20862        assert!(
20863            runner
20864                .guarded_pr_message(&winner, None, true)
20865                .await
20866                .unwrap()
20867                .is_none()
20868        );
20869        let record = runner.state.github_text.as_ref().unwrap();
20870        assert!(record.rewrite.is_none());
20871        // Records written before the additive field deserialize with no rewrite.
20872        let mut old = serde_json::to_value(record).unwrap();
20873        old.as_object_mut().unwrap().remove("rewrite");
20874        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
20875        assert!(old.rewrite.is_none());
20876        let id = record.question.clone().unwrap();
20877        ask::Questions::open()
20878            .update(&id, |q| {
20879                q.choices.push(crate::github_text::USE_REWRITE.into());
20880                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20881            })
20882            .unwrap();
20883        let message = runner
20884            .guarded_pr_message(&winner, None, true)
20885            .await
20886            .unwrap()
20887            .unwrap();
20888        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20889        assert_eq!(
20890            runner.state.github_text.as_ref().unwrap().choice,
20891            Some(GithubTextChoice::Fallback)
20892        );
20893    }
20894
20895    #[tokio::test]
20896    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
20897        crate::run::pin_test_home();
20898        let dir = tempfile::tempdir().unwrap();
20899        let mut runner = runner_at(RunStatus::Gating);
20900        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
20901        runner.state.repo = dir.path().to_owned();
20902        runner.state.candidates[0].worktree = dir.path().to_owned();
20903        seed_public_summary(&mut runner);
20904        let winner = runner.state.candidates[0].clone();
20905        assert!(
20906            runner
20907                .guarded_pr_message(&winner, None, true)
20908                .await
20909                .unwrap()
20910                .is_none()
20911        );
20912        let store = ask::Questions::open();
20913        let find = |run: &str| {
20914            store
20915                .list()
20916                .into_iter()
20917                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
20918                .unwrap()
20919        };
20920        let run = runner.state.id.clone();
20921        let q = find(&run);
20922        let fp = runner
20923            .state
20924            .github_text
20925            .as_ref()
20926            .unwrap()
20927            .fingerprint
20928            .clone();
20929        let name = crate::github_text::artifact_name(&fp);
20930        let path = crate::run::artifact_path(&runner.state, &name);
20931        assert!(q.detail.contains(&path.display().to_string()));
20932        assert!(q.detail.contains("title-language: title"));
20933        let saved = std::fs::read_to_string(&path).expect("artifact written");
20934        assert!(saved.contains("日本語のタイトル"));
20935        assert!(q.cwd.is_none());
20936        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
20937        // A bad replacement is refused and asked about once more.
20938        let say = |id: &str, text: &str, choice: &str| {
20939            store
20940                .update(id, |q| {
20941                    q.thread.push(ask::Turn {
20942                        who: ask::Who::Operator,
20943                        body: text.to_owned(),
20944                        at: jiff::Timestamp::now(),
20945                        note: None,
20946                    });
20947                    q.answer(ask::Answer::Choice(choice.to_owned()))
20948                })
20949                .unwrap();
20950        };
20951        say(&q.id, "まだ日本語", "use my text");
20952        assert!(
20953            runner
20954                .guarded_pr_message(&winner, None, true)
20955                .await
20956                .unwrap()
20957                .is_none()
20958        );
20959        let second = find(&run);
20960        assert_ne!(second.id, q.id);
20961        assert!(second.detail.contains("did not pass"));
20962        // The retry still compares the original text; the rejected
20963        // replacement is named by category only.
20964        assert!(second.detail.contains("日本語のタイトル"));
20965        assert!(!second.detail.contains("まだ日本語"));
20966        assert!(second.detail.contains(&path.display().to_string()));
20967        say(&second.id, "fix: retry failed requests", "use my text");
20968        let message = runner
20969            .guarded_pr_message(&winner, None, true)
20970            .await
20971            .unwrap()
20972            .expect("a vetted replacement is posted");
20973        assert_eq!(message.title, "fix: retry failed requests");
20974        assert_eq!(
20975            runner
20976                .state
20977                .github_text
20978                .as_ref()
20979                .unwrap()
20980                .chosen_title
20981                .as_deref(),
20982            Some("fix: retry failed requests")
20983        );
20984        assert!(runner.state.github_text.as_ref().unwrap().resolved);
20985        // Resumed again: the saved decision is applied, nothing is re-asked.
20986        let again = runner
20987            .guarded_pr_message(&winner, None, true)
20988            .await
20989            .unwrap()
20990            .unwrap();
20991        assert_eq!(again.title, message.title);
20992        assert_eq!(
20993            store
20994                .list()
20995                .into_iter()
20996                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
20997                .count(),
20998            2
20999        );
21000    }
21001
21002    #[test]
21003    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
21004        // What magi itself writes stays English under any configured language,
21005        // so a future localisation of these headings fails here. (The agents'
21006        // own text is also checked by the posting gate.)
21007        let mut state = state_with_summary(
21008            "add retries",
21009            "TITLE: fix(web): batch reads\n- reads run.json once",
21010        );
21011        state.config.graph.language = "ja".to_owned();
21012        let m = pr_message(&state, 'A');
21013        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
21014        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
21015
21016        // The task is the operator's own text: it goes in untouched, and the
21017        // fallback title (no summary) may be in its language too.
21018        let task = "今回やってほしいこと: results projector を直す";
21019        let mut state = state_with_summary(task, "- no title line");
21020        state.config.graph.language = "ja".to_owned();
21021        let m = pr_message(&state, 'A');
21022        assert_eq!(
21023            m.title,
21024            format!("chore: land candidate A of run {}", state.id)
21025        );
21026        assert!(
21027            m.body.contains(&format!(
21028                "<summary>Original task</summary>\n\n{task}\n\n</details>"
21029            )),
21030            "{}",
21031            m.body
21032        );
21033    }
21034
21035    #[test]
21036    fn pr_message_scrubs_home_paths_and_addresses() {
21037        let state = state_with_summary(
21038            "fix it in /Users/someone/src/x",
21039            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
21040        );
21041        let m = pr_message(&state, 'A');
21042        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
21043            assert!(!m.body.contains(leak), "{}", m.body);
21044        }
21045        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
21046    }
21047
21048    #[test]
21049    fn pr_message_survives_a_task_that_closes_details() {
21050        let state = state_with_summary("a </details> b", "TITLE: fix: x");
21051        let m = pr_message(&state, 'A');
21052        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
21053    }
21054
21055    #[test]
21056    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
21057        let cmd = manual_merge_command(
21058            MergeStyle::Squash,
21059            Path::new("/repo"),
21060            "b",
21061            "fix: \"quoted\" $(x) `y`\n\nbody",
21062        );
21063        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
21064    }
21065
21066    #[test]
21067    fn manual_merge_command_matches_the_configured_style() {
21068        let repo = Path::new("/repo");
21069        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
21070
21071        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
21072        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
21073
21074        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
21075        assert_eq!(
21076            squash,
21077            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
21078             \"Merge magi run 0832 (candidate A)\""
21079        );
21080
21081        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
21082        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
21083    }
21084
21085    #[test]
21086    fn a_nudge_gets_a_quarter_of_the_budget() {
21087        // The judge and implement budgets magi ships with.
21088        assert_eq!(retry_budget(secs(1200), true), secs(300));
21089        assert_eq!(retry_budget(secs(3600), true), secs(900));
21090    }
21091
21092    #[test]
21093    fn a_resent_prompt_keeps_the_whole_budget() {
21094        // The seat kept no context, so the retry is the original job again and
21095        // shortening it would only guarantee a second failure.
21096        assert_eq!(retry_budget(secs(1200), false), secs(1200));
21097        assert_eq!(retry_budget(secs(60), false), secs(60));
21098    }
21099
21100    #[test]
21101    fn the_floor_never_exceeds_the_original_budget() {
21102        // A short configured timeout must not be *raised* by the floor: the
21103        // operator asked for a bound, and a retry may not outlast the attempt
21104        // it is retrying.
21105        assert_eq!(retry_budget(secs(60), true), secs(60));
21106        assert_eq!(retry_budget(secs(480), true), secs(120));
21107        assert_eq!(retry_budget(secs(0), true), secs(0));
21108    }
21109
21110    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
21111        agent::CommandEvidence {
21112            id: "item1".to_owned(),
21113            description: "cargo test".to_owned(),
21114            exit_code,
21115            result_summary: String::new(),
21116            source: "codex".to_owned(),
21117        }
21118    }
21119
21120    #[test]
21121    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
21122        // No evidence is not the same fact as unconfirmed evidence: a
21123        // backend with no adapter, or a reply that ran no commands at all,
21124        // must not be misread as carrying a dangling job.
21125        assert!(!has_unconfirmed_command(&[]));
21126    }
21127
21128    #[test]
21129    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
21130        // Deliberately not a check on the exit code's *value*: a fixer
21131        // legitimately runs something that fails mid-iteration before it
21132        // succeeds, and that must never by itself reopen a valid report.
21133        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
21134        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
21135        assert!(!has_unconfirmed_command(&[
21136            evidence(Some(0)),
21137            evidence(Some(101))
21138        ]));
21139    }
21140
21141    #[test]
21142    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
21143        assert!(has_unconfirmed_command(&[
21144            evidence(Some(0)),
21145            evidence(None)
21146        ]));
21147    }
21148
21149    #[test]
21150    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
21151        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21152        assert_eq!(
21153            verified_noop_claim(true, &[], text).as_deref(),
21154            Some("already fixed by b32cfc4, on main.")
21155        );
21156    }
21157
21158    #[test]
21159    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
21160        // A timeout or a bad exit code reads as the ordinary loss it is,
21161        // whatever the reply's own prose claims.
21162        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21163        assert!(verified_noop_claim(false, &[], text).is_none());
21164    }
21165
21166    #[test]
21167    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
21168        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21169        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
21170        // A confirmed command alongside the marker is fine.
21171        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
21172    }
21173
21174    #[test]
21175    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
21176        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
21177    }
21178
21179    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
21180    /// pair, in order, labelled A, B, C, ...
21181    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
21182        runner.state.candidates = shape
21183            .iter()
21184            .enumerate()
21185            .map(|(i, &(empty, verified))| Candidate {
21186                index: i,
21187                label: (b'A' + i as u8) as char,
21188                agent: "sonnet".to_owned(),
21189                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
21190                worktree: PathBuf::from(format!("/wt/{i}")),
21191                summary: String::new(),
21192                stat: String::new(),
21193                files: 0,
21194                commits: 0,
21195                empty,
21196                failed: None,
21197                verified_noop: verified.map(str::to_owned),
21198                duration_ms: 0,
21199                folded: false,
21200            })
21201            .collect();
21202    }
21203
21204    #[test]
21205    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
21206        ask_test_home();
21207        let mut runner = runner_at(RunStatus::Implementing);
21208        set_candidates(
21209            &mut runner,
21210            &[
21211                (true, Some("already on main at b32cfc4")),
21212                (true, Some("same fix, see the existing test")),
21213            ],
21214        );
21215
21216        runner
21217            .after_implement()
21218            .expect("a verified no-op is not an error");
21219
21220        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
21221    }
21222
21223    #[test]
21224    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
21225        ask_test_home();
21226        let mut runner = runner_at(RunStatus::Implementing);
21227        // Candidate A declares a verified no-op; candidate B simply wrote
21228        // nothing and said nothing about why. One candidate's claim is not
21229        // the whole run's agreement.
21230        set_candidates(
21231            &mut runner,
21232            &[(true, Some("already on main at b32cfc4")), (true, None)],
21233        );
21234
21235        let err = runner
21236            .after_implement()
21237            .expect_err("an unverified empty candidate must still fail the run");
21238
21239        assert!(
21240            err.to_string().contains("no candidate produced a change"),
21241            "{err}"
21242        );
21243        assert_eq!(runner.state.status, RunStatus::Failed);
21244    }
21245
21246    #[test]
21247    fn after_implement_still_fails_an_ordinary_all_empty_run() {
21248        ask_test_home();
21249        let mut runner = runner_at(RunStatus::Implementing);
21250        set_candidates(&mut runner, &[(true, None), (true, None)]);
21251
21252        let err = runner
21253            .after_implement()
21254            .expect_err("no candidate declared anything; this is an ordinary failure");
21255
21256        assert!(
21257            err.to_string().contains("no candidate produced a change"),
21258            "{err}"
21259        );
21260        assert_eq!(runner.state.status, RunStatus::Failed);
21261    }
21262}
21263
21264/// A graft rollback whose restore could not be confirmed (see
21265/// [`Runner::graft`]).
21266#[derive(Debug)]
21267struct GraftRestoreFailed(String);
21268
21269impl std::fmt::Display for GraftRestoreFailed {
21270    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
21271        write!(
21272            f,
21273            "could not restore the winner to its pre-graft commit: {}",
21274            self.0
21275        )
21276    }
21277}
21278
21279impl std::error::Error for GraftRestoreFailed {}
21280
21281/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
21282/// not the agent: they never draw on an agent's own timeout, and running out of
21283/// them never reads as an agent failure.
21284#[cfg(test)]
21285mod resource_wait_tests {
21286    use super::*;
21287    use std::collections::BTreeMap;
21288
21289    fn sh(id: &str, script: &str) -> AgentSpec {
21290        AgentSpec {
21291            id: id.to_owned(),
21292            kind: crate::config::AgentKind::Command,
21293            model: None,
21294            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
21295            extra_args: Vec::new(),
21296            env: BTreeMap::new(),
21297            prompt_delivery: None,
21298            max_turn_secs: None,
21299        }
21300    }
21301
21302    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
21303        SeatJob {
21304            seat: SeatState::new("impl-A", &spec.id, 1),
21305            spec,
21306            cwd: dir.to_path_buf(),
21307            prompt: "do it".to_owned(),
21308            timeout,
21309            allow_write: true,
21310            sessions: false,
21311            artifacts: dir.join("art"),
21312            stem: "impl-A".to_owned(),
21313            handover: None,
21314        }
21315    }
21316
21317    fn state(resource_wait_secs: u64) -> RunState {
21318        let mut config = Config::default();
21319        config.graph.timeout_resource_wait = resource_wait_secs;
21320        RunState::new(
21321            PathBuf::from("/repo"),
21322            "main".to_owned(),
21323            "abc1234".to_owned(),
21324            "task".to_owned(),
21325            config,
21326        )
21327    }
21328
21329    fn hold(cache: &Path) -> crate::cache::Guard {
21330        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
21331        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
21332            crate::cache::AcquireOutcome::Acquired(g) => g,
21333            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
21334        }
21335    }
21336
21337    async fn run(
21338        jobs: Vec<SeatJob>,
21339        sem: Arc<Semaphore>,
21340        cache: Option<&Path>,
21341        st: &mut RunState,
21342    ) -> Vec<(usize, SeatState, AgentOutcome)> {
21343        let prompts = Prompts::default();
21344        let ctx = WaveCtx {
21345            run: "r",
21346            node: "implement",
21347            prompts: &prompts,
21348            cache,
21349            round: None,
21350            carry_seats: false,
21351            pauses: None,
21352        };
21353        wave(jobs, sem, &ctx, st, 0).await
21354    }
21355
21356    #[tokio::test]
21357    async fn verification_cache_wait_preserves_the_command_budget() {
21358        crate::run::pin_test_home();
21359        let tmp = tempfile::tempdir().unwrap();
21360        let cache = tmp.path().join("target");
21361        let guard = hold(&cache);
21362        let release = tokio::spawn(async move {
21363            tokio::time::sleep(Duration::from_millis(100)).await;
21364            guard.release();
21365        });
21366        let mut st = state(10);
21367        let budget = Duration::from_millis(50);
21368        let wait = repository_wait(&st, None);
21369        let (outcomes, _) = with_cache_lease(
21370            &mut st,
21371            Some(&cache),
21372            "gate",
21373            "gate",
21374            tmp.path(),
21375            "abc1234",
21376            budget,
21377            "test gate",
21378            &wait,
21379            |_, available| async move {
21380                assert_eq!(
21381                    available, budget,
21382                    "contention must not shorten verification"
21383                );
21384                (Vec::new(), false, Vec::new())
21385            },
21386        )
21387        .await
21388        .unwrap();
21389        release.await.unwrap();
21390        assert!(outcomes.is_empty(), "{outcomes:?}");
21391        assert!(
21392            st.events
21393                .iter()
21394                .any(|event| event.message.contains("waiting for"))
21395        );
21396        assert!(st.resource_wait.is_none());
21397        // The verification body no longer owns the lease after returning.
21398        hold(&cache).release();
21399    }
21400
21401    #[tokio::test]
21402    async fn verification_cache_wait_expires_without_starting_commands() {
21403        crate::run::pin_test_home();
21404        let tmp = tempfile::tempdir().unwrap();
21405        let cache = tmp.path().join("target");
21406        let _held = hold(&cache);
21407        let mut st = state(0);
21408        let wait = repository_wait(&st, None);
21409        let (outcomes, _) = with_cache_lease(
21410            &mut st,
21411            Some(&cache),
21412            "gate",
21413            "gate",
21414            tmp.path(),
21415            "abc1234",
21416            Duration::from_secs(60),
21417            "test gate",
21418            &wait,
21419            |_, _| async { panic!("commands must not start without the cache lease") },
21420        )
21421        .await
21422        .unwrap();
21423        assert_eq!(outcomes.len(), 1);
21424        assert!(outcomes[0].resource_blocked);
21425        assert!(st.resource_wait.is_none());
21426        assert!(matches!(
21427            st.resource_block.unwrap().cause,
21428            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
21429        ));
21430    }
21431
21432    #[tokio::test]
21433    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
21434        crate::run::pin_test_home();
21435        let tmp = tempfile::tempdir().unwrap();
21436        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
21437        let cache = tmp.path().join("cache");
21438        let mut state = state(21600);
21439        let wait = repository_wait(&state, None);
21440        let (outcomes, _) = with_cache_lease(
21441            &mut state,
21442            Some(&cache),
21443            "gate",
21444            "gate",
21445            tmp.path(),
21446            "head",
21447            Duration::from_secs(60),
21448            "gate",
21449            &wait,
21450            |_, _| async { panic!("freshness failure must not start verification") },
21451        )
21452        .await
21453        .unwrap();
21454        assert_eq!(outcomes.len(), 1);
21455        assert!(outcomes[0].resource_blocked);
21456        let block = state.resource_block.unwrap();
21457        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
21458        assert!(!block.description().contains("timed out"));
21459    }
21460
21461    #[tokio::test]
21462    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
21463        crate::run::pin_test_home();
21464        let tmp = tempfile::tempdir().unwrap();
21465        let cache = tmp.path().join("cache");
21466        let _held = hold(&cache);
21467        let mut state = state(21600);
21468        let pause = Pause::new();
21469        let interrupt = Pause::new();
21470        let cause = ParkCause::Operator;
21471        pause.park_for(cause.clone());
21472        let prompts = Prompts::default();
21473        let ctx = WaveCtx {
21474            run: "test",
21475            node: "fix",
21476            prompts: &prompts,
21477            cache: Some(&cache),
21478            round: None,
21479            carry_seats: false,
21480            pauses: Some((&pause, &interrupt)),
21481        };
21482        let result = wave(
21483            vec![job(
21484                sh("writer", "exit 99"),
21485                tmp.path(),
21486                Duration::from_secs(1),
21487            )],
21488            Arc::new(Semaphore::new(1)),
21489            &ctx,
21490            &mut state,
21491            0,
21492        )
21493        .await;
21494        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
21495        assert_eq!(result[0].1.turns, 0);
21496        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21497        assert_eq!(
21498            state.resource_block.unwrap().cause,
21499            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
21500        );
21501        assert!(state.active.is_empty());
21502    }
21503
21504    #[tokio::test]
21505    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
21506        crate::run::pin_test_home();
21507        let tmp = tempfile::tempdir().unwrap();
21508        let cache = tmp.path().join("target-a");
21509        let guard = hold(&cache);
21510        // Held for longer than the agent's whole 1s budget.
21511        tokio::spawn(async move {
21512            tokio::time::sleep(Duration::from_millis(1500)).await;
21513            guard.release();
21514        });
21515        let mut st = state(60);
21516        let results = run(
21517            vec![job(
21518                sh("alpha", "echo ok"),
21519                tmp.path(),
21520                Duration::from_secs(1),
21521            )],
21522            Arc::new(Semaphore::new(1)),
21523            Some(&cache),
21524            &mut st,
21525        )
21526        .await;
21527        assert!(
21528            matches!(results[0].2, AgentOutcome::Ok(_)),
21529            "the wait must not be charged to the agent"
21530        );
21531        assert!(st.resource_block.is_none());
21532        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21533        assert!(st.resource_wait.is_none() && st.active.is_empty());
21534    }
21535
21536    #[tokio::test]
21537    #[cfg(unix)]
21538    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
21539        use std::os::fd::AsRawFd as _;
21540        crate::run::pin_test_home();
21541        let tmp = tempfile::tempdir().unwrap();
21542        crate::git::git(tmp.path(), &["init"]).await.unwrap();
21543        let common = tmp.path().join(".git");
21544        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
21545        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
21546        assert_eq!(
21547            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
21548            0
21549        );
21550        let mut st = state(1);
21551        st.config.coordination = crate::config::test_coordination();
21552        let mut reader = job(
21553            sh("reader", "echo must-not-run"),
21554            tmp.path(),
21555            Duration::from_secs(30),
21556        );
21557        reader.allow_write = false;
21558        let results = run(
21559            vec![reader.clone()],
21560            Arc::new(Semaphore::new(1)),
21561            None,
21562            &mut st,
21563        )
21564        .await;
21565        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21566        assert_eq!(results[0].1.turns, 0);
21567        assert!(FailClass::of(&results[0].2).is_none());
21568        assert_eq!(
21569            st.resource_block.as_ref().unwrap().kind,
21570            "repository coordination"
21571        );
21572        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21573        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21574        drop(external);
21575        let editor = crate::repo_lock::SeatLease::for_checkout(
21576            tmp.path(),
21577            true,
21578            crate::config::test_coordination().as_ref(),
21579        )
21580        .await
21581        .unwrap();
21582        let pause = Pause::new();
21583        let interrupt = Pause::new();
21584        let prompts = Prompts::default();
21585        let ctx = WaveCtx {
21586            run: "r",
21587            node: "review",
21588            prompts: &prompts,
21589            cache: None,
21590            round: None,
21591            carry_seats: false,
21592            pauses: Some((&pause, &interrupt)),
21593        };
21594        let (results, ()) = tokio::join!(
21595            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
21596            async {
21597                tokio::time::sleep(Duration::from_millis(50)).await;
21598                pause.park();
21599            }
21600        );
21601        assert!(
21602            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
21603        );
21604        assert_eq!(results[0].1.turns, 0);
21605        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21606        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21607        drop(editor);
21608    }
21609
21610    #[tokio::test]
21611    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
21612        crate::run::pin_test_home();
21613        let tmp = tempfile::tempdir().unwrap();
21614        let cache = tmp.path().join("target-b");
21615        let _held = hold(&cache);
21616        let mut st = state(1);
21617        let results = run(
21618            vec![job(
21619                sh("alpha", "echo ok"),
21620                tmp.path(),
21621                Duration::from_secs(1),
21622            )],
21623            Arc::new(Semaphore::new(1)),
21624            Some(&cache),
21625            &mut st,
21626        )
21627        .await;
21628        let (_, seat, out) = &results[0];
21629        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
21630        assert!(wave_blocked(&results));
21631        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
21632        assert_eq!(seat.turns, 0, "nothing was asked");
21633        assert!(
21634            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
21635            "no agent was spawned"
21636        );
21637        assert!(FailClass::of(out).is_none(), "not a failure class");
21638        let block = st.resource_block.as_ref().expect("recorded for the resume");
21639        assert_eq!(block.kind, "shared build cache");
21640        assert!(matches!(
21641            block.cause,
21642            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
21643        ));
21644        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21645        assert!(st.active.is_empty());
21646    }
21647
21648    #[tokio::test]
21649    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
21650        crate::run::pin_test_home();
21651        let tmp = tempfile::tempdir().unwrap();
21652        let mut st = state(1);
21653        let results = run(
21654            vec![job(
21655                sh("alpha", "echo ok"),
21656                tmp.path(),
21657                Duration::from_secs(1),
21658            )],
21659            Arc::new(Semaphore::new(0)),
21660            None,
21661            &mut st,
21662        )
21663        .await;
21664        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21665        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
21666        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
21667    }
21668
21669    #[tokio::test]
21670    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
21671        crate::run::pin_test_home();
21672        let tmp = tempfile::tempdir().unwrap();
21673        let mut st = state(60);
21674        let results = run(
21675            vec![job(
21676                sh("alpha", "sleep 30"),
21677                tmp.path(),
21678                Duration::from_secs(1),
21679            )],
21680            Arc::new(Semaphore::new(1)),
21681            None,
21682            &mut st,
21683        )
21684        .await;
21685        let out = &results[0].2;
21686        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
21687        let class = FailClass::of(out).expect("a failure");
21688        assert_eq!(class, FailClass::Timeout);
21689        assert!(should_hand_over(None, &class), "existing handover policy");
21690        assert!(st.resource_block.is_none());
21691    }
21692}