Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
534/// on an early return or a panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Recorded before the first save, so a crash right after minting
773        // cannot leave a run with no origin. Legibility only: nothing reads it
774        // to decide anything.
775        state.fetched_base = Some(fetched_base);
776        state.origin = Some(origin);
777        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
778        state.status = RunStatus::Prep;
779        state.event("start", format!("run {} created", state.id));
780        startup.finish(&mut state, Ok(()))?;
781        Ok(Self {
782            state,
783            roles,
784            sem: Arc::new(Semaphore::new(max_parallel)),
785            pause: Pause::new(),
786            interrupt: Pause::new(),
787        })
788    }
789
790    /// Open a review-only run against work that already exists on `branch`.
791    ///
792    /// The expensive half of the graph is the implement wave — measured at
793    /// 111 and 134 internal tool-loop turns on this repository, against a
794    /// handful for a judge or a reviewer. The cheap half is worth running on
795    /// hand-written work too, and there was no way to reach it.
796    ///
797    /// No new state and no schema change are needed: a run with **one** viable
798    /// candidate and a tally already decided degrades `execute` to exactly
799    /// review → gate → merge, because `judge` skips a single-candidate field,
800    /// `deliberate` has fewer than two first choices to reconcile, `vote`
801    /// returns early, `tally` is already present and `fold_losers` has no
802    /// losers. Resuming such a run therefore does the right thing as well.
803    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
804        Self::review_taking_over(repo, branch, config, None, origin).await
805    }
806
807    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
808    /// at the same task still has `branch` checked out, its worktree is
809    /// released first if that is safe (see [`crate::handover`]), and the
810    /// review refuses with the reason if it is not. `None` is a hand-run
811    /// review: it has no earlier attempts, so only a worktree of a dead run
812    /// magi recorded itself can be released.
813    pub async fn review_taking_over(
814        repo: &Path,
815        branch: &str,
816        config: Config,
817        takeover: Option<crate::handover::Takeover>,
818        origin: Origin,
819    ) -> Result<Self> {
820        let repo = git::toplevel(repo).await?;
821        let missing = agent::missing_programs(&config.agents);
822        if !missing.is_empty() {
823            bail!(
824                "these agent programs are not on PATH: {}. Fix the roster in \
825                 magi.toml or install them.",
826                missing.join(", ")
827            );
828        }
829        let base_branch =
830            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
831                .await?;
832        if base_branch == branch {
833            bail!("`{branch}` is the base branch; there is nothing to review against");
834        }
835        let fetched_base = resolve_base(
836            &repo,
837            &base_branch,
838            &config.merge.remote,
839            config.coordination.as_ref(),
840        )
841        .await?;
842        let base_commit = fetched_base.commit.clone();
843
844        let roles = config.resolve_roles()?;
845        let max_parallel = config.graph.max_parallel.max(1);
846        let mut state = RunState::new(
847            repo.clone(),
848            base_branch,
849            base_commit.clone(),
850            format!("Review the work already on branch `{branch}`."),
851            config,
852        );
853        state.fetched_base = Some(fetched_base);
854        state.origin = Some(origin);
855        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
856        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
857        let opened = async {
858            startup.stage(&mut state, "handover")?;
859
860            // Released before anything else touches the branch: a stale local
861            // branch is moved with `git branch -f`, which git refuses while an
862            // earlier attempt's worktree still has it checked out. Everything
863            // after this point that can fail puts the old run back.
864            // A hand-run review has no task, hence no earlier attempts, but a
865            // worktree of a dead run magi made may still be released.
866            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
867                earlier: Vec::new(),
868                home: crate::run::home(),
869                choice: None,
870                task_instruction: None,
871            });
872            let wait = repository_wait(&state, None);
873            let coordination_config = state.config.coordination.clone();
874            let coordination =
875                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
876                    .await?;
877            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
878            if let Some(released) = &released {
879                state.event(
880                    "release",
881                    format!(
882                        "took `{branch}` over from run {}: its worktree was released: {}",
883                        crate::run::short_of(&released.old_id),
884                        released.audit
885                    ),
886                );
887            }
888            let opened = async {
889                startup.stage(&mut state, "review-preflight")?;
890                // The owner's answer to an earlier divergence question is applied
891                // here: after the release (git will not move a checked-out branch)
892                // and before the sync that would otherwise ask again.
893                state.foreign_checkout = crate::handover::preflight_review(
894                    &repo,
895                    branch,
896                    &takeover.home,
897                    &state.config.merge.remote,
898                )
899                .await?;
900                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
901                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
902                }
903                if let Some(choice) = takeover.choice.as_ref()
904                    && let Err(e) =
905                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
906                            .await
907                {
908                    return Err(e.context("applying the owner's answer about the diverged branch"));
909                }
910                // An earlier attempt of this task that published the task head, and
911                // whose work is the branch under review, hands that head over: the
912                // review run updates the same pull request. Without one (a legacy
913                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
914                // own head and its own open pull request is adopted as before.
915                for id in takeover.earlier.iter().rev() {
916                    if let Ok(old) = RunState::load_under(id, &takeover.home)
917                        && let Some(head) = old.pr_head.clone()
918                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
919                    {
920                        state.pr_head = Some(head);
921                        break;
922                    }
923                }
924                drop(coordination);
925                startup.stage(&mut state, "open-review")?;
926                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
927            }
928            .await;
929            if opened.is_err()
930                && let Some(released) = &released
931            {
932                // Restoration is ownership checked and best effort. Keep the original
933                // startup cause even if reacquiring coordination is no longer possible.
934                match crate::repo_lock::SeatLease::operation_with(
935                    &repo,
936                    coordination_config.as_ref(),
937                    &wait,
938                )
939                .await
940                {
941                    Ok(_coordination) => {
942                        released.restore(&repo, branch).await;
943                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
944                            .map(|old| {
945                                format!(
946                                    "predecessor released_to={:?}; candidate path exists={}",
947                                    old.released_to,
948                                    old.candidates.iter().any(|c| {
949                                        c.branch == branch && c.worktree.exists()
950                                    }),
951                                )
952                            })
953                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
954                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
955                    }
956                    Err(error) => {
957                        tracing::warn!(
958                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
959                            state.id
960                        );
961                        state.event(
962                            "startup-restore",
963                            format!("restoration not attempted: {error:#}"),
964                        );
965                    }
966                }
967            }
968            opened
969        }
970        .await;
971        startup.finish(&mut state, opened)?;
972        Ok(Self {
973            state,
974            roles,
975            sem: Arc::new(Semaphore::new(max_parallel)),
976            pause: Pause::new(),
977            interrupt: Pause::new(),
978        })
979    }
980
981    /// Recover exactly the operator-selected public implementation, without touching old sources.
982    pub async fn recover_pr(
983        repo: &Path,
984        task: &str,
985        selection: crate::pr_recovery::Selection,
986        mut config: Config,
987    ) -> Result<Self> {
988        selection.configure(&mut config)?;
989        selection.check_task(task, &selection.instruction)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.origin = Some(crate::run::Origin::queue(task));
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize exactly one additional rebase, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true).await
1357    }
1358
1359    async fn authorize_base_recovery_request(
1360        home: &Path,
1361        id: &str,
1362        task: &str,
1363        resumption: bool,
1364    ) -> Result<()> {
1365        let _owner = crate::pr_head::owner_lock(home, id)?;
1366        let mut state = RunState::load_under(id, home)?;
1367        crate::pr_head::ensure_owner(&state)?;
1368        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369        anyhow::ensure!(
1370            queued.runs.last().is_some_and(|run| run == id)
1371                && queued.status != crate::queue::TaskStatus::Done
1372                && !queued.fresh_start
1373                && queued.review_branch.is_none(),
1374            "run task ownership changed"
1375        );
1376        anyhow::ensure!(
1377            state
1378                .origin
1379                .as_ref()
1380                .and_then(|o| o.task.as_deref())
1381                .is_none_or(|owner| owner == task),
1382            "run belongs to another task"
1383        );
1384        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385        anyhow::ensure!(
1386            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387            "run is live or its ownership is unknown"
1388        );
1389        // Judge the previous driver's lease before acquiring our maintenance
1390        // lease. The owner lock excludes new driver claims across both steps.
1391        let _execution = crate::driver::maintenance(home, id)?;
1392        anyhow::ensure!(
1393            state.status.resumable()
1394                && state.released_to.is_none()
1395                && state.retired_to.is_none()
1396                && state.pr_taken_over_by.is_none(),
1397            "run ownership is final"
1398        );
1399        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400        anyhow::ensure!(
1401            !state.pr_publication_pending
1402                && !crate::handover::landing_checkpoint(&state)
1403                && state.merge_decision.is_none(),
1404            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405        );
1406        let winner = state.winner().context("run has no retained winner")?;
1407        anyhow::ensure!(
1408            git::is_clean(&winner.worktree).await?,
1409            "retained worktree has uncommitted changes"
1410        );
1411        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412        anyhow::ensure!(
1413            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414            "retained worktree does not match the winner branch"
1415        );
1416        state.record_base_sync_stop(&head);
1417        state.save()?;
1418        if resumption {
1419            anyhow::ensure!(
1420                crate::daemon::compatible_run_instruction(&state, &queued),
1421                "resumption instruction does not match the retained run"
1422            );
1423            let approval = queued
1424                .resumption
1425                .as_ref()
1426                .context("no operator resumption")?;
1427            anyhow::ensure!(
1428                approval.instruction == queued.instruction,
1429                "resumption approval does not match the current task instruction"
1430            );
1431            state.authorize_base_sync_resumption(task, approval)?;
1432        } else {
1433            state.authorize_base_sync(task)?;
1434        }
1435        state.event(
1436            "base-recovery",
1437            "owner authorized one additional base rebase; fresh verification required",
1438        );
1439        state.save()?;
1440        Ok(())
1441    }
1442
1443    /// Reopen an existing run.
1444    pub fn resume(id: &str) -> Result<Self> {
1445        let state = RunState::load(id)?;
1446        if state.startup.as_ref().is_some_and(|s| !s.complete)
1447            || (state.status == RunStatus::Prep
1448                && state.instruction.trim().is_empty()
1449                && state.candidates.is_empty()
1450                && state.prep_complete == Some(false))
1451        {
1452            bail!(
1453                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1454            );
1455        }
1456        crate::pr_head::ensure_owner(&state)?;
1457        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1458            bail!(
1459                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1460            );
1461        }
1462        if let Some(to) = &state.released_to {
1463            bail!(
1464                "run {} cannot be resumed: its worktree was released to run {}",
1465                state.short(),
1466                crate::run::short_of(to)
1467            );
1468        }
1469        let roles = state.config.resolve_roles()?;
1470        if let Some(id) = &state.config.graph.selected_implementer
1471            && state.candidates.iter().any(|c| c.agent != *id)
1472        {
1473            bail!(
1474                "replacement implementer pin does not match the saved candidates; task must stay held"
1475            );
1476        }
1477        let max_parallel = state.config.graph.max_parallel.max(1);
1478        Ok(Self {
1479            state,
1480            roles,
1481            sem: Arc::new(Semaphore::new(max_parallel)),
1482            pause: Pause::new(),
1483            interrupt: Pause::new(),
1484        })
1485    }
1486
1487    /// Walk the graph to a terminal state, skipping nodes already recorded.
1488    ///
1489    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1490    /// the phone - ends here, so this is the one place a run that ended
1491    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1492    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1493    pub async fn execute(&mut self) -> Result<()> {
1494        anyhow::ensure!(
1495            self.state.startup.as_ref().is_none_or(|s| s.complete),
1496            "startup incomplete; inspect retained evidence before explicit recovery"
1497        );
1498        if let Some(selection) = &self.state.pr_recovery
1499            && (selection.source_proof.is_some() || selection.source_run.is_some())
1500        {
1501            Box::pin(selection.verify_source(&self.state.repo)).await?;
1502        }
1503        // Queue claims cover daemon work; this also serializes manual drivers
1504        // consuming a repair answer, including the new review and decision.
1505        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1506            || self.state.land_approval.is_some()
1507            || self.state.merge_decision.is_some()
1508            || !self.state.merge_repairs.is_empty()
1509            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1510        {
1511            let claim = FixClaim::acquire(&self.state.dir())?;
1512            let disk = RunState::load(&self.state.id)?;
1513            let checkpoint = |s: &RunState| {
1514                (
1515                    s.merge_repairs.len(),
1516                    s.merge_repairs
1517                        .last()
1518                        .map(|r| (r.question.clone(), r.phase)),
1519                    s.land_approval
1520                        .as_ref()
1521                        .map(|a| (a.question.clone(), a.head.clone())),
1522                )
1523            };
1524            if checkpoint(&disk) != checkpoint(&self.state) {
1525                bail!(
1526                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1527                );
1528            }
1529            Some(claim)
1530        } else {
1531            None
1532        };
1533        let mut execution = self.claim_driver().await?;
1534        let runner = &mut *execution.runner;
1535        let wait = runner.repository_wait();
1536        // Keep the graph's large future off the callers' stack, including
1537        // standalone task attempts running on a default-sized test thread.
1538        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1539        let result = match result {
1540            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1541                record_repository_block(&mut runner.state, &e);
1542                runner.park_here()?;
1543                Ok(())
1544            }
1545            other => other,
1546        };
1547        execution.driver.finish(&mut runner.state).await?;
1548        let ended = if result.is_err() {
1549            Some({
1550                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1551                if let Err(error) = &result {
1552                    notice.message = format!(
1553                        "Run {} stopped with an error: {}",
1554                        runner.state.id,
1555                        error.to_string().lines().next().unwrap_or("unknown error")
1556                    );
1557                }
1558                notice
1559            })
1560        } else {
1561            crate::notices::run_ended(&runner.state)
1562        };
1563        if let Some(notice) = ended {
1564            crate::notices::raise(notice);
1565        }
1566        result
1567    }
1568
1569    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1570        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1571    }
1572
1573    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1574        let wait = self.repository_wait();
1575        coordinate_operation(&mut self.state, cwd, &wait).await
1576    }
1577
1578    /// Validate ownership and persist the driver claim in one critical section.
1579    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1580        let owner = match crate::pr_head::owner_lock_wait(
1581            &crate::run::home(),
1582            &self.state.id,
1583            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1584        )
1585        .await
1586        {
1587            Ok(owner) => owner,
1588            Err(error) => {
1589                let detail = format!("worktree maintenance or ownership update: {error:#}");
1590                // Admission did not own run.json: retain this resource outcome
1591                // in memory for queue settlement without overwriting a concurrent
1592                // retirement or publication transaction.
1593                self.state.resource_block = Some(crate::run::ResourceBlock {
1594                    cause: crate::run::ResourceBlockCause::Other,
1595                    node: "start".into(),
1596                    kind: "worktree maintenance".into(),
1597                    detail: detail.clone(),
1598                    at: Timestamp::now(),
1599                });
1600                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1601            }
1602        };
1603        let disk = RunState::load(&self.state.id)?;
1604        crate::pr_head::ensure_owner(&disk)?;
1605        let claimed =
1606            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1607        // Only proven-live ownership refuses here. Unknown records (legacy runs
1608        // with no claim marker or process identity) fall through to the
1609        // exclusive execution lease taken just below, which is what actually
1610        // keeps a second driver out.
1611        anyhow::ensure!(
1612            disk.liveness(claimed) != crate::run::Liveness::Live,
1613            "run already has a live driver; wait for its execution to end"
1614        );
1615        if let Some(to) = &disk.released_to {
1616            bail!(
1617                "run {} cannot continue: its worktree was released to run {}",
1618                self.state.short(),
1619                crate::run::short_of(to)
1620            );
1621        }
1622        anyhow::ensure!(
1623            serde_json::to_value(&disk.base_sync_recovery)?
1624                == serde_json::to_value(&self.state.base_sync_recovery)?,
1625            "base recovery changed since this run was loaded; reload the saved run"
1626        );
1627        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1628        let mut execution = DriverExecution {
1629            runner: self,
1630            driver,
1631        };
1632        // Declare after the execution guard: on any claim error this short
1633        // lock must drop before that guard reloads and releases ownership.
1634        let _owner = owner;
1635        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1636            .await?;
1637        let state = &mut execution.runner.state;
1638        // Moving again, so it is no longer parked. Set before the walk rather
1639        // than in `resume`, so every way of re-entering the graph clears it
1640        // and a card cannot claim a run is waiting to be resumed while the
1641        // agents are already working.
1642        state.parked = false;
1643        state.resource_wait = None;
1644        state.resource_block = None;
1645        // Any seat this state still lists as answering belongs to whatever
1646        // process last drove this run — this one included, if it crashed
1647        // mid-wave. Cleared and flushed immediately, before anything else
1648        // runs, so a resume can never show a seat as live when nothing is
1649        // asking it anything yet; the node that actually dispatches the next
1650        // wave repopulates it.
1651        state.clear_active();
1652        // Recorded in the same spot, and flushed together with the clear
1653        // above: this is the pid a reader checks (`RunState::liveness`) when
1654        // no daemon claim exists to answer "is a process still driving this
1655        // run" — a plain `magi run` / `magi review` typed into a terminal
1656        // claims nothing there. Always overwritten, never only-if-absent, so
1657        // a resumed run's stale pid from a previous, possibly-dead process
1658        // can never survive into this one's own report. Unlike
1659        // `clear_active`, this changes on every single `execute()` call, so
1660        // the save below is now unconditional rather than only-if-cleared.
1661        //
1662        // `driver_started_at` is recorded in the same breath, from this same
1663        // pid, so `liveness` can tell a live pid that is genuinely still us
1664        // apart from one the OS has since handed to an unrelated process —
1665        // see that field's own doc for why the pid alone is not enough.
1666        let pid = std::process::id();
1667        state.driver_pid = Some(pid);
1668        state.driver_started_at = crate::proc::process_started_at(pid);
1669        state.driver_exited = false;
1670        state.driver_claimed = Some(true);
1671        state.begin_pass();
1672        execution.driver.bind(state);
1673        state.save()?;
1674        execution.driver.record(state)?;
1675        Ok(execution)
1676    }
1677
1678    /// A grant owns only these repair attempts. Confirmation reviews cannot
1679    /// create another fixer allowance, and every operation is restart-checkpointed.
1680    async fn execute_granted_repair(&mut self) -> Result<()> {
1681        use crate::repair_budget::{Kind, Phase};
1682        let index = self.state.repair_grants.len() - 1;
1683        let winner = self
1684            .state
1685            .winner()
1686            .cloned()
1687            .context("repair winner missing")?;
1688        loop {
1689            let grant = self.state.repair_grants[index].clone();
1690            match grant.phase {
1691                Phase::Pending => {
1692                    if grant.spent >= grant.receipt.request.rounds {
1693                        self.state.repair_grants[index].phase = Phase::Complete;
1694                        self.state.status = RunStatus::Blocked;
1695                        self.state.save()?;
1696                        return Ok(());
1697                    }
1698                    // Reserve before dispatch. A crash can consume an attempt,
1699                    // but can never duplicate it or reset saved fixer history.
1700                    self.state.repair_grants[index].spent += 1;
1701                    self.state.repair_grants[index].phase = Phase::Fixing;
1702                    self.state.save()?;
1703                    match grant.receipt.request.kind {
1704                        Kind::Gate => {
1705                            let failed = self.state.gate.clone();
1706                            let _ = self.gate_fix_round(&winner, &failed).await?;
1707                        }
1708                        Kind::Review => {
1709                            let mut record = self
1710                                .state
1711                                .reviews
1712                                .last()
1713                                .cloned()
1714                                .context("review failure missing")?;
1715                            let round = record.round;
1716                            let step = self.run_fix_round(&winner, &mut record).await?;
1717                            self.state.reviews[round - 1] = record;
1718                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1719                                self.state.repair_grants[index].phase = Phase::Complete;
1720                                self.state.status = RunStatus::Blocked;
1721                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1722                                self.state.save()?;
1723                                return Ok(());
1724                            }
1725                            if matches!(step, FixStep::Resource) {
1726                                self.stop_for_resource()?;
1727                                return Ok(());
1728                            }
1729                            self.state.save()?;
1730                        }
1731                    }
1732                    if self.resource_halt()? {
1733                        return Ok(());
1734                    }
1735                }
1736                Phase::Fixing => {
1737                    // Interrupted fixers may have left source changes or an
1738                    // unpushed commit. Preserve them before fresh assessment.
1739                    let _coordination = self.coordinate(&winner.worktree).await?;
1740                    let rescued = git::rescue_commit(
1741                        &winner.worktree,
1742                        "magi: preserve interrupted granted repair",
1743                    )
1744                    .await?;
1745                    self.state.note_withheld("repair-grant", &rescued.withheld);
1746                    // Fresh checks must supersede even a same-head failure.
1747                    // Archive rather than erase failure evidence or counters.
1748                    self.state
1749                        .verification_history
1750                        .push(crate::run::VerificationEvidence {
1751                            superseded_by: None,
1752                            archived_at: Some(Timestamp::now()),
1753                            fixes_spent: Some(self.state.gate_fixes.len()),
1754                            gate_head: self.state.gate_head.take(),
1755                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1756                            gate: std::mem::take(&mut self.state.gate),
1757                            e2e: self.state.head_e2e.take(),
1758                        });
1759                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1760                        self.state
1761                            .verification_history
1762                            .push(crate::run::VerificationEvidence {
1763                                superseded_by: None,
1764                                archived_at: Some(Timestamp::now()),
1765                                fixes_spent: Some(self.state.gate_fixes.len()),
1766                                gate_head: None,
1767                                gate_ran: false,
1768                                gate: Vec::new(),
1769                                e2e: Some(e2e),
1770                            });
1771                    }
1772                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1773                    self.state.repair_grants[index].head =
1774                        git::rev_parse(&winner.worktree, "HEAD").await?;
1775                    self.state.repair_grants[index].phase = Phase::Reviewing;
1776                    self.state.save()?;
1777                }
1778                Phase::Reviewing => {
1779                    self.review_loop_pass().await?;
1780                    if self.resource_halt()? {
1781                        return Ok(());
1782                    }
1783                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1784                    let clean = self.state.reviews.len() > grant.review_start
1785                        && self.state.reviews.last().is_some_and(|r| {
1786                            r.head == head
1787                                && r.clean
1788                                && r.expected > 0
1789                                && r.answered == r.expected
1790                                && matches!(
1791                                    r.e2e_status(),
1792                                    E2eStatus::Passed | E2eStatus::NotConfigured
1793                                )
1794                        });
1795                    if !clean {
1796                        // A confirmation review carries no additional fix budget.
1797                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1798                            == Kind::Review
1799                            && grant.spent < grant.receipt.request.rounds
1800                            && self.state.reviews.last().is_some_and(|r| {
1801                                r.fix.is_none()
1802                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1803                            }) {
1804                            Phase::Pending
1805                        } else {
1806                            Phase::Complete
1807                        };
1808                        self.state.status = RunStatus::Blocked;
1809                        self.state.event(
1810                            "repair-grant",
1811                            "fresh review did not clear the repaired head; no publication",
1812                        );
1813                        self.state.save()?;
1814                        if self.state.repair_grants[index].phase == Phase::Pending {
1815                            continue;
1816                        }
1817                        return Ok(());
1818                    }
1819                    self.state.repair_grants[index].phase = Phase::Gating;
1820                    self.state.save()?;
1821                }
1822                Phase::Gating => {
1823                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1824                    self.state.reconcile_verification_head(&head);
1825                    let wait = self.repository_wait();
1826                    if self
1827                        .state
1828                        .reviews
1829                        .last()
1830                        .is_none_or(|r| r.head != head || !r.clean)
1831                    {
1832                        self.state.status = RunStatus::Blocked;
1833                        self.state.repair_grants[index].phase = Phase::Complete;
1834                        self.state.event(
1835                            "repair-grant",
1836                            "head changed since fresh review; repair held",
1837                        );
1838                        self.state.save()?;
1839                        return Ok(());
1840                    }
1841                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1842                        return Ok(());
1843                    }
1844                    let outcomes = self.run_gate(&winner).await?;
1845                    if verify_inconclusive(&outcomes) {
1846                        return Ok(());
1847                    }
1848                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1849                    if after != head {
1850                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1851                        self.state.repair_grants[index].head = after;
1852                        self.state.repair_grants[index].phase = Phase::Reviewing;
1853                        self.state.status = RunStatus::Blocked;
1854                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1855                        self.state.save()?;
1856                        return Ok(());
1857                    }
1858                    let passed = outcomes.iter().all(CommandOutcome::ok);
1859                    self.state.repair_grants[index].succeeded = passed;
1860                    self.state.gate = outcomes;
1861                    self.state.gate_ran = true;
1862                    self.state.status = if passed {
1863                        RunStatus::Gating
1864                    } else {
1865                        RunStatus::Blocked
1866                    };
1867                    self.state.repair_grants[index].phase = if !passed
1868                        && grant.receipt.request.kind == Kind::Gate
1869                        && grant.spent < grant.receipt.request.rounds
1870                    {
1871                        Phase::Pending
1872                    } else {
1873                        Phase::Complete
1874                    };
1875                    if passed && self.state.config.merge.mode == MergeMode::Pr {
1876                        // Persist publication debt with the successful checkpoint.
1877                        self.state.pr_publication_pending = true;
1878                    } else if passed {
1879                        self.state.status = RunStatus::Ready;
1880                    }
1881                    self.state.save()?;
1882                    if !passed {
1883                        continue;
1884                    }
1885                    // Existing PR ownership and push lease stay in place. The
1886                    // normal publisher owes the new reviewed/gated SHA, and
1887                    // land's fresh owner approval remains mandatory.
1888                    if self.state.config.merge.mode != MergeMode::Pr {
1889                        self.state.status = RunStatus::Ready;
1890                        self.state.event(
1891                            "repair-grant",
1892                            "repair verified locally; grant carries no merge authorization",
1893                        );
1894                        self.state.save()?;
1895                        return Ok(());
1896                    }
1897                    return self.merge().await;
1898                }
1899                Phase::Complete => return Ok(()),
1900            }
1901        }
1902    }
1903
1904    async fn execute_graph(&mut self) -> Result<()> {
1905        self.state.configure_recovery_resume()?;
1906        if self
1907            .state
1908            .repair_grants
1909            .last()
1910            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
1911        {
1912            return self.execute_granted_repair().await;
1913        }
1914        if self
1915            .state
1916            .repair_grants
1917            .last()
1918            .is_some_and(|g| !g.succeeded)
1919        {
1920            self.state.status = RunStatus::Blocked;
1921            self.state.save()?;
1922            return Ok(());
1923        }
1924        // Preservation may invalidate an old empty classification. Resolve it
1925        // before any resume shortcut can reuse a verdict or publish a branch.
1926        self.persist_candidates().await?;
1927        // A saved base stop outranks stage markers, including legacy Voting.
1928        // Recovery never reenters implementation, deliberation or voting.
1929        if self.state.base_sync_blocked() {
1930            self.sync_to_base().await?;
1931            if !self.state.base_sync_blocked()
1932                && self.state.status != RunStatus::AlreadyInBase
1933                && !self.resource_halt()?
1934            {
1935                self.finish_after_tally().await?;
1936            }
1937            return Ok(());
1938        }
1939        if self
1940            .state
1941            .merge_decision
1942            .as_ref()
1943            .is_some_and(|decision| !decision.closed)
1944            && self.state.status != RunStatus::Landing
1945        {
1946            if crate::merge_decision::resumable_hold(&self.state) {
1947                self.state.merge = Some(MergeOutcome {
1948                    mode: MergeMode::Pr,
1949                    ok: false,
1950                    detail: self
1951                        .state
1952                        .merge_decision
1953                        .as_ref()
1954                        .unwrap()
1955                        .target
1956                        .pr
1957                        .clone(),
1958                    empty: false,
1959                });
1960                self.run_land().await?;
1961                self.settle_questions();
1962                return Ok(());
1963            }
1964            bail!(
1965                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1966            );
1967        }
1968        if self.state.config.graph.selected_implementer.is_some()
1969            && self.state.status == RunStatus::Blocked
1970            && self.state.tally.is_none()
1971            && self.state.candidate_preservation.is_empty()
1972            && self.state.interrupted_implementations.is_empty()
1973        {
1974            bail!(
1975                "selected implementation is held; explicitly release the owning task to retry it"
1976            );
1977        }
1978
1979        if self.state.merge_repairs.last().is_some_and(|r| {
1980            matches!(
1981                r.phase,
1982                crate::run::MergeRepairPhase::Requested
1983                    | crate::run::MergeRepairPhase::Fixing
1984                    | crate::run::MergeRepairPhase::Reviewing
1985            )
1986        }) {
1987            if self.repair_merge_review().await? {
1988                self.run_land().await?;
1989            }
1990            self.settle_questions();
1991            return Ok(());
1992        }
1993        // Publication already passed review and gate. Resume only that node:
1994        // syncing to a newer base here could change the verified commit.
1995        if self.state.pr_publication_pending {
1996            self.merge().await?;
1997            if self.resource_halt()? {
1998                return Ok(());
1999            }
2000            self.state.save()?;
2001            self.settle_questions();
2002            return Ok(());
2003        }
2004        // A run that already lost its quorum never resumes into the verdict
2005        // machinery: `deliberate` and `vote` would otherwise clobber the
2006        // stalled marker back to Voting and the run would keep going past a
2007        // verdict that is no longer trustworthy. Everything already recorded is
2008        // kept, so the run stays resumable (or foldable) for a human to pick up.
2009        //
2010        // On --resume the run gets one chance to repair itself: the seats a
2011        // rate limit took out are re-asked. If their quota has since reset and
2012        // the quorum is restored, the run picks up and finishes; otherwise it
2013        // stays stale and still-resumable for a later retry. If it does not
2014        // recover, the returned status stays `Stalled` and nothing was
2015        // clobbered (the recovery only mutates entries for the lost seats).
2016        if self.state.status == RunStatus::Stalled {
2017            if self.recover_stall().await? {
2018                self.finish_after_tally().await?;
2019            } else {
2020                // Still below quorum: persist the marker and stay resumable.
2021                self.state.save()?;
2022            }
2023            return Ok(());
2024        }
2025        // A run parked inside `land` - watching CI, mid fix-round, or
2026        // waiting on the owner's merge approval - resumes directly into it,
2027        // never back through `prep`. Everything before `merge` already
2028        // concluded; that is the only way `status` reaches `Landing` in the
2029        // first place. Re-walking `review_loop` first would also be actively
2030        // wrong: its own status recomputation (see its doc) treats any
2031        // clean round as reason to set `status` to `Gating`, which would
2032        // clobber this marker before `merge` ever ran, and this run would
2033        // never find its way back into `land` at all.
2034        if self.state.status == RunStatus::Landing
2035            || crate::handover::landing_checkpoint(&self.state)
2036        {
2037            self.state.land_sync_pending = true;
2038            self.state.save()?;
2039            self.run_land().await?;
2040            // `run_land` may have settled the run right here - CI came back
2041            // green and the PR merged, say - without ever passing back
2042            // through `merge`'s own trailing call. Whatever it left `status`
2043            // as is what this has to read.
2044            self.settle_questions();
2045            return Ok(());
2046        }
2047        // A run parked at `merge` on the owner's word about a withheld PR text
2048        // resumes straight into `merge`. Re-walking the tail would sync to the
2049        // base again (possibly rebasing, possibly spending the fixer) after the
2050        // gate had passed, and then push a tree nobody reviewed or gated.
2051        if self.state.github_text.is_some()
2052            && self.state.merge.is_none()
2053            && self.state.status == RunStatus::Gating
2054        {
2055            self.merge().await?;
2056            if self.resource_halt()? {
2057                return Ok(());
2058            }
2059            self.state.save()?;
2060            return Ok(());
2061        }
2062        if let Err(e) = self.prep().await {
2063            if e.is::<crate::repo_lock::ResourceBlocked>() {
2064                return Err(e);
2065            }
2066            self.state.status = RunStatus::Blocked;
2067            self.state
2068                .prep_failure
2069                .get_or_insert_with(|| format!("{e:#}"));
2070            self.state
2071                .event("prep", format!("preparation stopped: {e:#}"));
2072            self.state.save()?;
2073            return Err(e);
2074        }
2075        if self.park_here()? {
2076            return Ok(());
2077        }
2078        self.advise().await?;
2079        if self.park_here()? {
2080            return Ok(());
2081        }
2082        self.implement().await?;
2083        if self.park_here()? {
2084            return Ok(());
2085        }
2086        // `after_implement` already saved the state and settled any open
2087        // questions when it set this; nothing later in the graph has
2088        // anything to judge.
2089        if self.state.status == RunStatus::VerifiedNoop {
2090            return Ok(());
2091        }
2092        self.judge().await?;
2093        if self.park_here()? {
2094            return Ok(());
2095        }
2096        self.deliberate().await?;
2097        if self.park_here()? {
2098            return Ok(());
2099        }
2100        self.vote().await?;
2101        if self.park_here()? {
2102            return Ok(());
2103        }
2104        self.tally()?;
2105        // A verdict that lost its quorum is not trustworthy: do not review,
2106        // gate, or merge on it. Everything already done is kept, so the run
2107        // stays resumable (or foldable); the human can replace the agent that
2108        // ran out of quota and pick it up.
2109        if self.state.status == RunStatus::Stalled {
2110            // Persist the stalled marker now — the normal end-of-execute save
2111            // below is below this early return, and without it a resumed run
2112            // would reload a pre-tally status and keep going.
2113            self.state.save()?;
2114            return Ok(());
2115        }
2116        self.finish_after_tally().await?;
2117        Ok(())
2118    }
2119
2120    /// A wave could not start its agents because a shared resource stayed busy
2121    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2122    /// resumable. Unstarted seats have no result; completed peers retain
2123    /// their saved candidate state.
2124    fn stop_for_resource(&mut self) -> Result<()> {
2125        // Resource acquisition observes cancellation inside a node. Complete
2126        // the same durable park as every other node boundary before returning.
2127        if self.pause.parked() || self.interrupt.parked() {
2128            self.park_here()?;
2129            return Ok(());
2130        }
2131        if self.state.resource_block.is_some() {
2132            self.state.status = RunStatus::Blocked;
2133        }
2134        self.state.save()
2135    }
2136
2137    /// Stop walking the graph if a node ended on a resource block.
2138    fn resource_halt(&mut self) -> Result<bool> {
2139        if self.state.resource_block.is_none() {
2140            return Ok(false);
2141        }
2142        self.stop_for_resource()?;
2143        Ok(true)
2144    }
2145
2146    /// Park here if asked to, recording it in the run's own timeline.
2147    ///
2148    /// Returns whether the caller should stop walking the graph. The state is
2149    /// saved either way by the node that just finished; this adds the event so
2150    /// the operator's card says why a run that is neither finished nor moving
2151    /// is sitting where it is.
2152    fn park_here(&mut self) -> Result<bool> {
2153        let parking = self.pause.parked() || self.interrupt.parked();
2154        if !parking && self.state.resource_block.is_some() {
2155            self.stop_for_resource()?;
2156            return Ok(true);
2157        }
2158        // Either handle asking is enough - see `Pause`'s own doc for why
2159        // they are never the same one. `interrupt` is checked second so a
2160        // reason it carries is preferred in the message below over a plain
2161        // shutdown park racing it at the same boundary.
2162        if !self.pause.parked() && !self.interrupt.parked() {
2163            return Ok(false);
2164        }
2165        if self.state.parked {
2166            return Ok(true);
2167        }
2168        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2169        if let Some(block) = &mut self.state.resource_block
2170            && matches!(
2171                block.cause,
2172                crate::run::ResourceBlockCause::Cancelled { .. }
2173            )
2174        {
2175            block.cause = crate::run::ResourceBlockCause::Cancelled {
2176                park: cause.clone(),
2177            };
2178        }
2179        let why = match cause.as_ref().map(ParkCause::label) {
2180            Some(reason) => format!(
2181                "parked after `{}` ({reason}) — resume to carry on from here",
2182                self.state.status.as_str()
2183            ),
2184            None => format!(
2185                "parked after `{}` — resume to carry on from here",
2186                self.state.status.as_str()
2187            ),
2188        };
2189        self.state.event("park", why);
2190        self.state
2191            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2192        self.state.parked = true;
2193        self.state.save()?;
2194        Ok(true)
2195    }
2196
2197    /// Only a new explicit operator approval may retry failed preparation.
2198    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2199        if self.state.prep_failure.is_none() {
2200            return Ok(());
2201        }
2202        let failed_at = self
2203            .state
2204            .events
2205            .iter()
2206            .rev()
2207            .find(|event| event.node == "prep")
2208            .map_or(self.state.updated_at, |event| event.at);
2209        anyhow::ensure!(
2210            approval_at > failed_at,
2211            "preparation retry requires a new explicit release after the failure"
2212        );
2213        self.state.prep_failure = None;
2214        self.state.prep_complete = Some(false);
2215        self.state
2216            .event("prep-retry", "operator approved retry of preparation");
2217        Ok(())
2218    }
2219
2220    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2221    pub fn on_pause(&mut self, pause: Pause) {
2222        self.pause = pause;
2223    }
2224
2225    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2226    /// scheduler asking this one run - and no other - to park so a task
2227    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2228    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2229    /// handle.
2230    pub fn watch_interrupt(&mut self, pause: Pause) {
2231        self.interrupt = pause;
2232    }
2233
2234    /// Abandon this run's own open questions, once `status` has actually
2235    /// settled rather than merely paused.
2236    ///
2237    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2238    /// either back up with the candidates, the review round and the seat
2239    /// sessions already on disk, so a question an implementer asked mid-round
2240    /// may still get a real answer read by a real resume. Only the statuses
2241    /// `resumable` excludes are actually final: the run merged, it reached
2242    /// `Ready` with nothing left to do, it failed outright with no
2243    /// established point to continue from, or every candidate agreed, with
2244    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2245    /// every one of those the seat that asked is gone for good, exactly like
2246    /// the run being deleted under `magi run rm` - so the same cleanup
2247    /// applies, worded for what actually happened instead of "the run was
2248    /// deleted".
2249    ///
2250    /// Best-effort and silent on success: called from every place `status`
2251    /// can land on one of those three, including ones a resumed run revisits,
2252    /// so it must cost nothing when there was nothing open to begin with.
2253    fn settle_questions(&mut self) {
2254        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2255            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2256        }
2257    }
2258
2259    /// The tail of the graph after a trustworthy tally: fold losers, review,
2260    /// gate, merge, and persist.
2261    async fn finish_after_tally(&mut self) -> Result<()> {
2262        // Before the losers are folded: their branches are what the graft
2263        // reads from.
2264        self.graft().await?;
2265        if self.resource_halt()? {
2266            return Ok(());
2267        }
2268        self.fold_losers().await?;
2269        // Before review starts, and again right before the gate: a run's
2270        // review rounds can themselves take long enough for the base to move
2271        // a second time, and the gate is the one node whose "green" gets
2272        // acted on.
2273        self.sync_to_base().await?;
2274        if self.hold_base_sync().await? || self.resource_halt()? {
2275            return Ok(());
2276        }
2277        if self.state.status == RunStatus::AlreadyInBase {
2278            return Ok(());
2279        }
2280        self.review_loop().await?;
2281        if self.resource_halt()? {
2282            return Ok(());
2283        }
2284        self.sync_to_base().await?;
2285        if self.hold_base_sync().await? || self.resource_halt()? {
2286            return Ok(());
2287        }
2288        if self.state.status == RunStatus::AlreadyInBase {
2289            return Ok(());
2290        }
2291        self.gate().await?;
2292        self.merge().await?;
2293        if self.resource_halt()? {
2294            return Ok(());
2295        }
2296        self.state.save()?;
2297        Ok(())
2298    }
2299
2300    // ---------------------------------------------------------------- prep
2301
2302    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2303    /// run with the step and its output in the event; it is never swallowed.
2304    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2305    /// resume starts over instead of finding candidates and missing seats.
2306    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2307        let result = match ensure_setup_config(&mut self.state, repo).await {
2308            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2309            Err(e) => Err(e),
2310        };
2311        if let Err(e) = result {
2312            git::worktree_remove_clean(repo, wt).await.ok();
2313            if in_prep {
2314                self.rollback_prep(repo).await;
2315            }
2316            self.state.status = RunStatus::Blocked;
2317            self.state.event(
2318                "setup",
2319                format!("worktree setup failed in {}: {e:#}", wt.display()),
2320            );
2321            self.state.save()?;
2322            return Err(e);
2323        }
2324        Ok(())
2325    }
2326
2327    /// Undo everything `prep` created, so a resume re-runs it whole.
2328    async fn rollback_prep(&mut self, repo: &Path) {
2329        let root = self.state.worktree_root();
2330        for c in std::mem::take(&mut self.state.candidates) {
2331            if matches!(
2332                git::worktree_remove_clean(repo, &c.worktree).await,
2333                Ok(true)
2334            ) {
2335                git::branch_delete(repo, &c.branch).await.ok();
2336            } else {
2337                self.state.candidates.push(c);
2338            }
2339        }
2340        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2341        for j in 1..=seats {
2342            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2343                .await
2344                .ok();
2345        }
2346        for k in 1..=self.state.config.graph.advisors {
2347            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2348                .await
2349                .ok();
2350        }
2351    }
2352
2353    async fn prep(&mut self) -> Result<()> {
2354        if let Some(reason) = &self.state.prep_failure {
2355            bail!(
2356                "candidate preparation previously failed: {reason}; correct the source before retrying"
2357            );
2358        }
2359        if self
2360            .state
2361            .prep_complete
2362            .unwrap_or(!self.state.candidates.is_empty())
2363        {
2364            return Ok(());
2365        }
2366        let repo = self.state.repo.clone();
2367        // Includes rollback, config, refs, checkout creation and setup. No
2368        // repository mutation may precede the real shared lock acquisition.
2369        let _coordination = self.coordinate(&repo).await?;
2370        let base = self.state.base_commit.clone();
2371        let plan = self.state.sources.plan(&repo, &base).await?;
2372        self.rollback_prep(&repo).await;
2373        if !self.state.candidates.is_empty() {
2374            bail!(
2375                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2376            );
2377        }
2378        self.state.prep_complete = Some(false);
2379        self.state.status = RunStatus::Prep;
2380        self.state.save()?;
2381        let repo = self.state.repo.clone();
2382        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2383        let root = self.state.worktree_root();
2384        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2385
2386        // The hook is the write-time half of the blindness contract; the
2387        // presentation filter in `blind` is the half that cannot be bypassed.
2388        let hooks_dir = self.state.dir().join("hooks");
2389        if self.state.config.blind.commit_msg_hook {
2390            std::fs::create_dir_all(&hooks_dir)
2391                .with_context(|| format!("create {}", hooks_dir.display()))?;
2392            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2393            let path = hooks_dir.join("commit-msg");
2394            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2395            make_executable(&path)?;
2396            // Ref-counted rather than a plain idempotent set: with more than
2397            // one run able to be in flight in the same repository at once
2398            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2399            // true?" check cannot tell "another run of mine still needs
2400            // this" from "nobody does", and the run that happens to finish
2401            // first would disable the hook out from under a sibling still
2402            // relying on it.
2403            git::acquire_worktree_config(&repo).await?;
2404            self.state.enabled_worktree_config = true;
2405        }
2406
2407        // `[worktree] setup` hides its products per worktree, which needs the
2408        // same per-worktree config the hook does. Held until fold, like it.
2409        ensure_setup_config(&mut self.state, &repo).await?;
2410
2411        for (index, (spec, label)) in self
2412            .roles
2413            .implementers
2414            .clone()
2415            .into_iter()
2416            .zip(labels)
2417            .enumerate()
2418        {
2419            let branch = self.state.branch_for(label);
2420            let worktree = root.join(format!("cand-{label}"));
2421            // Persist intent before Git can leave a partially created tree.
2422            // Candidate registration follows setup, so it cannot inventory
2423            // crashes or refusals during creation itself.
2424            self.state.event(
2425                "prep-intent",
2426                serde_json::json!({
2427                    "branch": branch,
2428                    "worktree": worktree,
2429                    "start": start,
2430                    "picks": plan.picks,
2431                })
2432                .to_string(),
2433            );
2434            self.state.save()?;
2435            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2436            if self.state.config.blind.commit_msg_hook {
2437                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2438            }
2439            git::local_exclude(&worktree, "/.magi/").await?;
2440            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2441                // Nothing of this prep may survive: a resume must start over,
2442                // not find half the candidates and skip the rest.
2443                git::worktree_remove_clean(&repo, &worktree).await.ok();
2444                git::branch_delete(&repo, &branch).await.ok();
2445                self.rollback_prep(&repo).await;
2446                self.state.status = RunStatus::Blocked;
2447                self.state
2448                    .event("prep", format!("worktree setup failed: {e:#}"));
2449                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2450                self.state.save()?;
2451                return Err(e);
2452            }
2453            self.state.candidates.push(Candidate {
2454                index,
2455                label,
2456                agent: spec.id.clone(),
2457                branch,
2458                worktree,
2459                summary: String::new(),
2460                stat: String::new(),
2461                files: 0,
2462                commits: 0,
2463                empty: false,
2464                failed: None,
2465                verified_noop: None,
2466                duration_ms: 0,
2467                folded: false,
2468            });
2469            self.state.save()?;
2470            for pick in &plan.picks {
2471                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2472                {
2473                    self.state.status = RunStatus::Blocked;
2474                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2475                    self.state
2476                        .event("prep", format!("cannot apply referenced commit: {e}"));
2477                    self.state.save()?;
2478                    return Err(e);
2479                }
2480            }
2481        }
2482
2483        for j in 1..=self.roles.judges.len() {
2484            let wt = root.join(format!("judge-{j}"));
2485            if !wt.exists() {
2486                git::worktree_add_detached(&repo, &wt, &base).await?;
2487                self.setup_seat_worktree(&repo, &wt, true).await?;
2488            }
2489        }
2490
2491        // Disposable, detached checkouts for the design-deliberation stage's
2492        // advisor seats — the same shape as the judges' above, at the same
2493        // base commit, since advisors also only ever read. Sized off the
2494        // configured count directly rather than a resolved roster: unlike
2495        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2496        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2497        // `prep` has no `ResolvedRoles` field to read a count from here.
2498        if self.state.config.graph.advise {
2499            for k in 1..=self.state.config.graph.advisors {
2500                let wt = root.join(format!("advisor-{k}"));
2501                if !wt.exists() {
2502                    git::worktree_add_detached(&repo, &wt, &base).await?;
2503                    self.setup_seat_worktree(&repo, &wt, true).await?;
2504                }
2505            }
2506        }
2507
2508        // A judge cannot tell it is looking at its own patch — the seats keep
2509        // separate conversations — but a panel that shares agents with the
2510        // field is less independent than it looks, and that is worth saying out
2511        // loud once per run rather than leaving it in the config.
2512        let authors: Vec<&str> = self
2513            .roles
2514            .implementers
2515            .iter()
2516            .map(|a| a.id.as_str())
2517            .collect();
2518        let overlap: Vec<String> = self
2519            .roles
2520            .judges
2521            .iter()
2522            .enumerate()
2523            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2524            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2525            .collect();
2526        if !overlap.is_empty() {
2527            let note = format!(
2528                "{} also authored a candidate; blind, but the panel is less \
2529                 independent than {} distinct agents would be",
2530                overlap.join(", "),
2531                self.roles.judges.len()
2532            );
2533            self.state.event("prep", note);
2534        }
2535
2536        self.state.event(
2537            "prep",
2538            format!(
2539                "{} candidates, {} judges, base {} ({})",
2540                self.state.candidates.len(),
2541                self.roles.judges.len(),
2542                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2543                self.state.base_branch
2544            ),
2545        );
2546        self.state.prep_complete = Some(true);
2547        self.state.status = RunStatus::Implementing;
2548        self.state.save()?;
2549        Ok(())
2550    }
2551
2552    // -------------------------------------------------------------- advise
2553
2554    /// The design-deliberation stage: independent, read-only advisor seats
2555    /// each sketch a design before any implementer touches the repository,
2556    /// and (when at least one produced a usable proposal) a synthesis seat
2557    /// blends them into a brief `implement` carries in every candidate's
2558    /// prompt.
2559    ///
2560    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2561    /// advisors` is the proposal count. Everything here is best-effort and
2562    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2563    /// that cannot reach quota, or a synthesis seat that produced nothing
2564    /// usable all leave `implement` exactly as it was before this stage
2565    /// existed — the task instruction alone — rather than failing the whole
2566    /// competition over an enrichment stage. Every outcome is still recorded
2567    /// as an event, so a run that got nothing from this stage says why.
2568    ///
2569    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2570    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2571    /// resumed run whose stage failed would re-run it, and re-spend the
2572    /// agent calls, on every reentry before `implement`.
2573    ///
2574    /// Also skipped once any candidate shows implementation progress — the
2575    /// exact predicate `implement` itself uses to decide a candidate is no
2576    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2577    /// is not enough: a run created by an older binary that predates this
2578    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2579    /// an already-`Implementing`-or-later run under this build would
2580    /// otherwise walk straight back through `prep` (a no-op once candidates
2581    /// exist) into this node and spawn every advisor seat against worktrees
2582    /// `prep` never recreated — after implementation has already started,
2583    /// which is exactly the invariant this stage exists to guarantee.
2584    async fn advise(&mut self) -> Result<()> {
2585        let implement_untouched = self
2586            .state
2587            .candidates
2588            .iter()
2589            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2590        if !self.state.config.graph.advise || self.state.advise_attempted {
2591            return Ok(());
2592        }
2593        if !implement_untouched {
2594            self.state.event(
2595                "advise",
2596                "skipping the design-deliberation stage: at least one \
2597                 candidate already shows implementation progress, so this \
2598                 run is past the point the stage exists to run before"
2599                    .to_owned(),
2600            );
2601            self.state.advise_attempted = true;
2602            self.state.save()?;
2603            return Ok(());
2604        }
2605        let run_id = self.state.id.clone();
2606        let prompts = self.state.config.prompts.clone();
2607        let instruction = self.state.instruction.clone();
2608        let language = self.state.config.graph.language.clone();
2609        let root = self.state.worktree_root();
2610        let n = self.state.config.graph.advisors;
2611        let where_recorded = self.state.dir().join("run.json");
2612
2613        let seats = match self.state.config.advisors() {
2614            Ok(seats) if !seats.is_empty() => seats,
2615            Ok(_) => {
2616                self.state.event(
2617                    "advise",
2618                    format!(
2619                        "[graph] advisors is 0; skipping the design-deliberation \
2620                         stage and continuing without a synthesis brief (see {})",
2621                        where_recorded.display()
2622                    ),
2623                );
2624                self.state.advise_attempted = true;
2625                self.state.save()?;
2626                return Ok(());
2627            }
2628            Err(e) => {
2629                self.state.event(
2630                    "advise",
2631                    format!(
2632                        "could not resolve advisor seats ({e:#}); continuing \
2633                         without a design-deliberation brief (see {})",
2634                        where_recorded.display()
2635                    ),
2636                );
2637                self.state.advise_attempted = true;
2638                self.state.save()?;
2639                return Ok(());
2640            }
2641        };
2642
2643        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2644        let artifacts = agent::artifacts_dir(&self.state.dir());
2645        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2646
2647        let mut jobs = Vec::new();
2648        for (i, spec) in seats.iter().cloned().enumerate() {
2649            let seat_key = format!("advisor-{}", i + 1);
2650            let seat = self.seat(&seat_key, &spec.id);
2651            jobs.push(SeatJob {
2652                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2653                spec,
2654                seat,
2655                cwd: worktrees[i % worktrees.len()].clone(),
2656                timeout,
2657                allow_write: false,
2658                sessions: false,
2659                artifacts: artifacts.clone(),
2660                stem: seat_key,
2661                handover: None,
2662            });
2663        }
2664
2665        self.state.event(
2666            "advise",
2667            format!(
2668                "{} advisor seat(s) sketching a design in parallel",
2669                jobs.len()
2670            ),
2671        );
2672        let mut quota_losses = Vec::new();
2673        let cache = self.state.config.cache_dir();
2674        let ctx = WaveCtx {
2675            carry_seats: false,
2676            pauses: Some((&self.pause, &self.interrupt)),
2677            run: &run_id,
2678            node: "advise",
2679            prompts: &prompts,
2680            cache: cache.as_deref(),
2681            round: None,
2682        };
2683        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2684        let results = ask_json_wave::<Proposal>(
2685            jobs,
2686            Arc::clone(&self.sem),
2687            self.state.config.graph.retries,
2688            &advisor_roster,
2689            &ctx,
2690            &mut quota_losses,
2691            &mut self.state,
2692            &|p: &Proposal| p.validate(),
2693        )
2694        .await;
2695        if self.state.resource_block.is_some() {
2696            return self.stop_for_resource();
2697        }
2698        self.state.quota.extend(quota_losses);
2699
2700        let mut records = Vec::with_capacity(results.len());
2701        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2702            let agent_id = seat.agent.clone();
2703            self.state.seats.insert(seat.key.clone(), seat);
2704            match res {
2705                Ok((proposal, out)) => {
2706                    self.state
2707                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2708                    records.push(advise::AdvisorRecord::proposed(
2709                        i + 1,
2710                        agent_id,
2711                        proposal,
2712                        out.duration_ms,
2713                    ));
2714                }
2715                Err(e) => {
2716                    self.state.event(
2717                        "advise",
2718                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2719                    );
2720                    records.push(advise::AdvisorRecord::failed(
2721                        i + 1,
2722                        agent_id,
2723                        e.to_string(),
2724                    ));
2725                }
2726            }
2727        }
2728
2729        let mut advice = advise::Advice {
2730            records,
2731            synthesis: None,
2732        };
2733        if advice.proposals().is_empty() {
2734            self.state.event(
2735                "advise",
2736                "no advisor produced a usable proposal; continuing without a \
2737                 synthesis brief"
2738                    .to_owned(),
2739            );
2740        } else {
2741            match self
2742                .synthesize_brief(
2743                    &advice,
2744                    &instruction,
2745                    &language,
2746                    &worktrees[0],
2747                    &artifacts,
2748                    &run_id,
2749                    &prompts,
2750                    cache.as_deref(),
2751                )
2752                .await
2753            {
2754                Ok(Some(text)) => {
2755                    self.state.event(
2756                        "advise",
2757                        "synthesized a design brief for the implementer".to_owned(),
2758                    );
2759                    advice.synthesis = Some(text);
2760                }
2761                Ok(None) => {
2762                    self.state.event(
2763                        "advise",
2764                        "the synthesis seat produced nothing usable; continuing \
2765                         without a design brief"
2766                            .to_owned(),
2767                    );
2768                }
2769                Err(e) => {
2770                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2771                        return Err(e);
2772                    }
2773                    self.state.event(
2774                        "advise",
2775                        format!("could not synthesize a design brief: {e:#}"),
2776                    );
2777                }
2778            }
2779        }
2780        advise::apply_reflection(&mut advice);
2781
2782        self.state.advice = Some(advice);
2783        self.state.advise_attempted = true;
2784        self.state.save()?;
2785        Ok(())
2786    }
2787
2788    /// The synthesis seat: reads every advisor's proposal and blends them
2789    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2790    /// out of [`Runner::advise`] only for readability — it is not called
2791    /// anywhere else.
2792    ///
2793    /// Picked the same way [`crate::talk`]'s standing conversation and
2794    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2795    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2796    /// order (a claude seat, else the first runnable agent in roster order)
2797    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2798    /// in [`crate::config`] for why a dedicated field exists here at all.
2799    #[allow(clippy::too_many_arguments)]
2800    async fn synthesize_brief(
2801        &mut self,
2802        advice: &advise::Advice,
2803        instruction: &str,
2804        language: &str,
2805        cwd: &Path,
2806        artifacts: &Path,
2807        run_id: &str,
2808        prompts: &Prompts,
2809        cache: Option<&Path>,
2810    ) -> Result<Option<String>> {
2811        let chain = agent::pick_chain(
2812            &self.state.config.agents,
2813            self.state.config.roles.synthesizer.as_ref(),
2814            &agent::installed,
2815            "synthesizer",
2816        )?;
2817        let proposals = advice.proposals();
2818        let mut prompt = prompt::with_overlay(
2819            prompt::synthesize_brief(instruction, &proposals, language),
2820            prompts.overlay("advise"),
2821        );
2822        if cache.is_some() {
2823            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2824            // below — see `prompt::build_cache_note`'s doc for why telling a
2825            // read-only seat to build through the shared cache is exactly how
2826            // a sandbox's write refusal gets misread as a defect.
2827            prompt.push('\n');
2828            prompt.push_str(&prompt::build_cache_note("advise", false));
2829        }
2830        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2831        // Each id is tried once, in order; a quota hit, error or unusable
2832        // answer moves to the next. The seat is single-turn (`sessions:
2833        // false`) and the prompt is the whole context, so a fallback agent
2834        // needs nothing carried over.
2835        let mut last = None;
2836        for (n, spec) in chain.iter().enumerate() {
2837            if n > 0 {
2838                self.state
2839                    .event("advise", format!("synthesis falling back to {}", spec.id));
2840            }
2841            let mut seat = self.seat("advise-synthesis", &spec.id);
2842            let wait = self.repository_wait();
2843            let outcome = wait
2844                .scope(agent::invoke(
2845                    spec,
2846                    &mut seat,
2847                    &Invocation {
2848                        coordination: self.state.config.coordination.as_ref(),
2849                        fetched_base: None,
2850                        cwd,
2851                        prompt: &prompt,
2852                        timeout,
2853                        allow_write: false,
2854                        sessions: false,
2855                        artifacts,
2856                        stem: &if n == 0 {
2857                            "advise-synthesis".to_owned()
2858                        } else {
2859                            format!("advise-synthesis-{}", spec.id)
2860                        },
2861                        run: run_id,
2862                        node: "advise",
2863                        cache_dir: None,
2864                        attachments: &[],
2865                        writable: &[],
2866                    },
2867                ))
2868                .await;
2869            if outcome.is_ok() {
2870                self.state.seats.insert(seat.key.clone(), seat);
2871            }
2872            let advance = agent::chain_advances(&outcome);
2873            last = Some(outcome);
2874            if !advance {
2875                break;
2876            }
2877        }
2878        // Exhausted: the last attempt's result is what a single failed seat
2879        // would have produced.
2880        let out = last.expect("a chain holds at least one agent")?;
2881        if !out.usable() {
2882            return Ok(None);
2883        }
2884        let text =
2885            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2886        Ok((!text.trim().is_empty()).then_some(text))
2887    }
2888
2889    // ----------------------------------------------------------- implement
2890
2891    async fn implement(&mut self) -> Result<()> {
2892        self.persist_candidates().await?;
2893        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2894        // agent files with `magi task add` name the run that paid for it. The
2895        // prompt overlay is cloned alongside it because the waves borrow it
2896        // while `self` is mutably borrowed by the node's own bookkeeping.
2897        let run_id = self.state.id.clone();
2898        let prompts = self.state.config.prompts.clone();
2899        let todo: Vec<usize> = self
2900            .state
2901            .candidates
2902            .iter()
2903            .enumerate()
2904            .filter(|(_, c)| {
2905                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2906                    && c.failed.is_none()
2907                    && !c.empty
2908            })
2909            .map(|(i, _)| i)
2910            .collect();
2911        if todo.is_empty() {
2912            return self.after_implement();
2913        }
2914        self.state.status = RunStatus::Implementing;
2915
2916        let language = self.state.config.graph.language.clone();
2917        let github_language = self.state.config.graph.github_language.clone();
2918        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2919        let sessions = self.state.config.graph.sessions;
2920        let artifacts = agent::artifacts_dir(&self.state.dir());
2921        // The design-deliberation stage's blended brief, when `advise` found
2922        // one — carried into every implementer's prompt the same way
2923        // regardless of which candidate it is.
2924        let brief = self
2925            .state
2926            .advice
2927            .as_ref()
2928            .and_then(|a| a.synthesis.as_deref())
2929            .map(str::to_owned);
2930        let attachments = self.state.attachments.clone();
2931
2932        let mut jobs = Vec::new();
2933        for &i in &todo {
2934            let (index, label, worktree) = {
2935                let c = &self.state.candidates[i];
2936                (c.index, c.label, c.worktree.clone())
2937            };
2938            let spec = self.roles.implementers[index].clone();
2939            let seat_key = format!("impl-{label}");
2940            let seat = self.seat(&seat_key, &spec.id);
2941            let instruction = seeded_instruction(&self.state);
2942            jobs.push(SeatJob {
2943                spec,
2944                seat,
2945                prompt: prompt::implement_in(
2946                    &instruction,
2947                    &worktree.to_string_lossy(),
2948                    &language,
2949                    &github_language,
2950                    brief.as_deref(),
2951                    &attachments,
2952                ),
2953                cwd: worktree,
2954                timeout,
2955                allow_write: true,
2956                sessions,
2957                artifacts: artifacts.clone(),
2958                stem: format!("impl-{label}"),
2959                handover: None,
2960            });
2961        }
2962
2963        self.state.event(
2964            "implement",
2965            format!("{} candidates in parallel", jobs.len()),
2966        );
2967        // Kept so a seat whose CLI hung up can be asked again from the same
2968        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2969        // can update a seat's own entry once a fallback agent takes it over —
2970        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2971        // whichever agent actually answered, not the one that quota'd out.
2972        let mut sent = jobs.clone();
2973        let cache = self.state.config.cache_dir();
2974        let ctx = WaveCtx {
2975            carry_seats: false,
2976            pauses: Some((&self.pause, &self.interrupt)),
2977            run: &run_id,
2978            node: "implement",
2979            prompts: &prompts,
2980            cache: cache.as_deref(),
2981            round: None,
2982        };
2983        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2984        if !wave_blocked(&results) {
2985            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2986                .await;
2987            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2988                .await?;
2989            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2990                .await;
2991        }
2992        // Do not finalize failed candidates when the entire wave needs machine
2993        // recovery: leaving them pending resumes this node with its work intact.
2994        if !results.is_empty()
2995            && results
2996                .iter()
2997                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
2998        {
2999            let losses: Vec<_> = results
3000                .iter()
3001                .map(|(_, seat, out)| {
3002                    let AgentOutcome::Quota(o) = out else {
3003                        unreachable!()
3004                    };
3005                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3006                })
3007                .collect();
3008            self.state.quota.extend(losses.clone());
3009            for (_, seat, _) in results {
3010                self.state.seats.insert(seat.key.clone(), seat);
3011            }
3012            for &i in &todo {
3013                let label = self.state.candidates[i].label;
3014                self.state.interrupted_implementations.insert(label);
3015                self.state
3016                    .candidate_preservation
3017                    .insert(label, "machine outage awaiting persistence".into());
3018            }
3019            // Preserve partial edits too, while keeping every interrupted seat
3020            // pending even when its tree already contains an inherited patch.
3021            block_machine_outage(&mut self.state, "implement", &losses);
3022            self.state.save()?;
3023            self.persist_candidates().await?;
3024            for &i in &todo {
3025                self.state
3026                    .interrupted_implementations
3027                    .insert(self.state.candidates[i].label);
3028            }
3029            return self.stop_for_resource();
3030        }
3031        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3032
3033        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3034            && results
3035                .iter()
3036                .any(|(_, _, out)| FailClass::of(out).is_some());
3037        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3038            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3039                // Executed turns interrupted by disk pressure already carry
3040                // a preservation barrier from wave. Admission-only waits do
3041                // not. Keep the interrupted seat's session for recovery too.
3042                if self
3043                    .state
3044                    .candidate_preservation
3045                    .contains_key(&self.state.candidates[i].label)
3046                {
3047                    self.state.seats.insert(seat.key.clone(), seat);
3048                }
3049                continue;
3050            }
3051            let seat_key = seat.key.clone();
3052            // A quota fallback (`resume_seat_handovers`) may have handed this
3053            // seat to a different agent than the one `prep` recorded on the
3054            // candidate; the stats tables and any later fixer-defaults-to-
3055            // winner's-author lookup must credit whoever actually answered —
3056            // unless every fallback also quota'd out, in which case nobody
3057            // actually answered and crediting the last agent tried would
3058            // erase every earlier agent's own quota loss from the stats
3059            // tables instead of just this one seat's.
3060            let agent = seat.agent.clone();
3061            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3062            self.state.seats.insert(seat.key.clone(), seat);
3063            let label = self.state.candidates[i].label;
3064            self.state.interrupted_implementations.remove(&label);
3065
3066            let (summary, duration, failed, verified_claim) = match out {
3067                AgentOutcome::Ok(o) => {
3068                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3069                    let failed = (!o.usable()).then(|| {
3070                        if o.timed_out {
3071                            "agent timed out".to_owned()
3072                        } else {
3073                            format!(
3074                                "agent exited with {}",
3075                                crate::notices::exit_status(o.exit_code)
3076                            )
3077                        }
3078                    });
3079                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3080                    (text, o.duration_ms, failed, verified_claim)
3081                }
3082                // Left un-resumed by `resume_undelivered` (a dirty tree
3083                // already rescues the work, or there was no session left to
3084                // resume into) — reported like the ordinary failure it is,
3085                // never as if `o.text` (the CLI's raw error JSON) were an
3086                // answer.
3087                AgentOutcome::Dropped(o) => {
3088                    let why = o
3089                        .dropped
3090                        .as_ref()
3091                        .map(|d| d.why.as_str())
3092                        .unwrap_or("the CLI ended the stream without delivering its answer");
3093                    (
3094                        String::new(),
3095                        o.duration_ms,
3096                        Some(format!("the CLI dropped the stream ({why})")),
3097                        None,
3098                    )
3099                }
3100                AgentOutcome::Quota(o) => {
3101                    self.state.quota.push(QuotaLoss::from_report(
3102                        seat_key,
3103                        "implement",
3104                        o.quota.as_ref(),
3105                    ));
3106                    (
3107                        String::new(),
3108                        o.duration_ms,
3109                        Some("rate limited (quota); produced no change".to_owned()),
3110                        None,
3111                    )
3112                }
3113                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3114                    (String::new(), 0, Some(e), None)
3115                }
3116            };
3117
3118            // Save every completed turn before any fallible Git operation. A failed
3119            // rescue must not erase the reply or prevent collection of other seats.
3120            let c = &mut self.state.candidates[i];
3121            if !exhausted_the_fallback_chain {
3122                c.agent = agent;
3123            }
3124            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3125            c.duration_ms = duration;
3126            c.failed = failed;
3127            c.verified_noop = verified_claim;
3128            self.state
3129                .candidate_preservation
3130                .insert(label, "awaiting persistence".to_owned());
3131        }
3132        self.state.save()?;
3133        self.persist_candidates().await?;
3134
3135        if resource_blocked {
3136            return self.stop_for_resource();
3137        }
3138        if selected_failed {
3139            self.state.status = RunStatus::Blocked;
3140            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3141            self.state.save()?;
3142            bail!(
3143                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3144            );
3145        }
3146        self.after_implement()
3147    }
3148
3149    /// Retry only persistence after a completed implementation turn.
3150    async fn persist_candidates(&mut self) -> Result<()> {
3151        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3152        for label in labels {
3153            let i = self
3154                .state
3155                .candidates
3156                .iter()
3157                .position(|c| c.label == label)
3158                .context("pending candidate is missing")?;
3159            let c = self.state.candidates[i].clone();
3160            let interrupted = self.state.interrupted_implementations.contains(&label);
3161            let result = async {
3162                let wait = crate::repo_lock::WaitPolicy::new(
3163                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3164                    || false,
3165                );
3166                let _coordination =
3167                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3168                // Best effort evidence requires no index writes. A successful snapshot
3169                // never authorizes removal of a tree whose rescue failed.
3170                if let Err(e) = git::snapshot_uncommitted(
3171                    &c.worktree,
3172                    &self
3173                        .state
3174                        .dir()
3175                        .join("artifacts")
3176                        .join(format!("candidate-{label}-recovery")),
3177                )
3178                .await
3179                {
3180                    self.state.event(
3181                        "preserve",
3182                        format!("candidate {label} snapshot incomplete: {e:#}"),
3183                    );
3184                }
3185                let rescued = git::rescue_commit(
3186                    &c.worktree,
3187                    &format!("magi: candidate {label} (uncommitted work)"),
3188                )
3189                .await?;
3190                self.state.note_withheld("implement", &rescued.withheld);
3191                let base = &self.state.base_commit;
3192                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3193                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3194                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3195                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3196                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3197                if c.empty && commits > 0 && !patch.trim().is_empty()
3198                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3199                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3200                        || self.state.tally.is_some())
3201                {
3202                    // Keep the old classification and pending marker so every
3203                    // retry stops too. Retain decisions as recovery evidence;
3204                    // only a supported recovery operation may replace them.
3205                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3206                }
3207                let c = &mut self.state.candidates[i];
3208                c.commits = commits;
3209                c.stat = stat;
3210                c.files = files;
3211                let no_changes = commits == 0 || patch.trim().is_empty();
3212                c.empty = no_changes && !interrupted;
3213                if interrupted && no_changes {
3214                    c.failed = None;
3215                    c.verified_noop = None;
3216                }
3217                if !c.empty {
3218                    c.failed = None;
3219                    c.verified_noop = None;
3220                }
3221                self.state.event(
3222                    "preserve",
3223                    format!("candidate {label}: {files} files, {commits} commits"),
3224                );
3225                Ok::<_, anyhow::Error>(())
3226            }
3227            .await;
3228            match result {
3229                Ok(()) => {
3230                    self.state.candidate_preservation.remove(&label);
3231                    if !interrupted || self.state.candidates[i].files > 0 {
3232                        self.state.interrupted_implementations.remove(&label);
3233                    }
3234                }
3235                Err(e) => {
3236                    let reason = format!("candidate {label} preservation failed: {e:#}");
3237                    self.state.event("preserve", &reason);
3238                    self.state.candidate_preservation.insert(label, reason);
3239                }
3240            }
3241            if self.state.candidate_preservation.is_empty()
3242                && self.state.status == RunStatus::Blocked
3243                && self.state.candidates.iter().all(|c| c.failed.is_none())
3244            {
3245                // Persist the recovered phase with the cleared pending marker;
3246                // a crash between saves must not turn this into a selected-seat
3247                // failure hold on the next resume.
3248                self.state.status = RunStatus::Implementing;
3249                self.state.event(
3250                    "preserve",
3251                    "candidate persistence recovered; implementation can continue",
3252                );
3253            }
3254            self.state.save()?;
3255        }
3256        if !self.state.candidate_preservation.is_empty() {
3257            self.state.status = RunStatus::Blocked;
3258            self.state.save()?;
3259            bail!(
3260                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3261            );
3262        }
3263        Ok(())
3264    }
3265
3266    /// Ask again, once, for work a CLI did and then failed to hand over.
3267    ///
3268    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3269    /// status with an empty response and a usage report showing output tokens,
3270    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3271    /// seven minutes and 14,267 output tokens that arrived as an empty
3272    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3273    ///
3274    /// Two conditions, and both matter:
3275    ///
3276    /// - **Only when the tree is untouched.** Often the agent has already
3277    ///   written its files and only the closing message was lost; the rescue
3278    ///   commit below picks that up and there is nothing to ask for. Re-asking
3279    ///   then would pay for a second implementation of work already on disk.
3280    /// - **Once.** A CLI that drops one stream can drop the next, and this
3281    ///   node is the most expensive in the graph.
3282    ///
3283    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3284    /// dropped reply still carried its `conversation_id`, so the seat is asked
3285    /// to finish what it was doing rather than sent the whole task again. It
3286    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3287    /// node's - for the same reason a re-ranked judge does: restating finished
3288    /// work is not the work.
3289    ///
3290    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3291    /// same way until it resets, while an abandoned conversation is still
3292    /// there to be picked up.
3293    async fn resume_undelivered(
3294        &mut self,
3295        results: &mut [(usize, SeatState, AgentOutcome)],
3296        sent: &[SeatJob],
3297        prompts: &Prompts,
3298        run_id: &str,
3299    ) {
3300        for (wi, seat, out) in results.iter_mut() {
3301            let Some(dropped) = (match &*out {
3302                AgentOutcome::Dropped(o) => o.dropped.clone(),
3303                _ => None,
3304            }) else {
3305                continue;
3306            };
3307            let Some(job) = sent.get(*wi) else { continue };
3308            // Already on disk? Then only the closing message was lost.
3309            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3310                self.state.event(
3311                    "implement",
3312                    format!(
3313                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3314                         work is in the tree",
3315                        seat.key, dropped.output_tokens, dropped.why
3316                    ),
3317                );
3318                continue;
3319            }
3320            // The re-ask only makes sense as a resume: `resume_after_drop`
3321            // says nothing about the task, trusting the seat to still hold it.
3322            // Without a session to resume — sessions disabled, or this CLI's
3323            // drop shape happened not to carry a session id — that prompt
3324            // would open a brand-new conversation with no context at all,
3325            // which is worse than leaving this as the ordinary failure it
3326            // already is.
3327            if !has_context(&job.spec, seat, job.sessions) {
3328                self.state.event(
3329                    "implement",
3330                    format!(
3331                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3332                         is no session left to resume",
3333                        seat.key, dropped.output_tokens, dropped.why
3334                    ),
3335                );
3336                continue;
3337            }
3338            self.state.event(
3339                "implement",
3340                format!(
3341                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3342                     conversation",
3343                    seat.key, dropped.output_tokens, dropped.why
3344                ),
3345            );
3346            let mut retry = job.clone();
3347            retry.seat = seat.clone();
3348            retry.prompt = prompt::resume_after_drop(&dropped.why);
3349            retry.timeout = retry_budget(job.timeout, true);
3350            retry.stem = format!("{}-resume", job.stem);
3351            let cache = self.state.config.cache_dir();
3352            let ctx = WaveCtx {
3353                carry_seats: false,
3354                pauses: Some((&self.pause, &self.interrupt)),
3355                run: run_id,
3356                node: "implement",
3357                prompts,
3358                cache: cache.as_deref(),
3359                round: None,
3360            };
3361            let (resumed_seat, resumed) =
3362                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3363            *seat = resumed_seat;
3364            *out = resumed;
3365        }
3366    }
3367
3368    /// Fall an implement seat through to the next untried agent in the
3369    /// implementer roster when it lost to quota — or, since the handover was
3370    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3371    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3372    /// quota path itself being unchanged — instead of leaving the
3373    /// seat's loss final the moment one agent's account runs dry.
3374    ///
3375    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3376    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3377    /// truncates to the single slot rotation picked, so a solo task whose one
3378    /// implementer hits quota mid-run used to have nothing else to try. This
3379    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3380    /// unrotated roster — which is the only place the *other* candidates in
3381    /// the machine's roster still exist once `implementers` has been cut down
3382    /// to size.
3383    ///
3384    /// Walks forward from just past the seat's own original position in the
3385    /// roster, never wrapping back to the front: a later candidate slot (say
3386    /// `beta`, the roster's second entry) must fall through to the *next*
3387    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3388    /// almost certainly a different candidate's own agent already — and once
3389    /// the roster's tail is exhausted there is nothing left to fall through
3390    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3391    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3392    /// let this retry that id forever. The loop keeps falling through until
3393    /// an attempt lands something other than `Quota` or the roster's tail
3394    /// runs out of untried ids, at which point the seat is left exactly as
3395    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3396    /// `QuotaLoss` recorded, the candidate failed/empty.
3397    ///
3398    /// `sent` is taken mutably and updated with the fallback agent's spec:
3399    /// `resume_unconfirmed_commands`, which runs after this and also reads
3400    /// `sent`, must see whichever agent actually ended up answering the seat
3401    /// — reading the stale, original spec there would check session
3402    /// eligibility against the wrong CLI and could hand a fallback agent's
3403    /// session id to the agent that just lost the seat to quota.
3404    ///
3405    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3406    /// — `self.seat` only reuses state when the agent id is unchanged, so
3407    /// handing it a different id already gets this for free. Reusing the old
3408    /// seat would resume a different CLI's session as if it were a
3409    /// continuation of this one.
3410    ///
3411    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3412    /// a quota loss cuts an agent off mid-turn, so anything already in the
3413    /// tree is unfinished work, not a completed candidate a re-ask would pay
3414    /// for twice. A dirty tree is rescued into a commit first (the same
3415    /// neutral-identity rescue `implement`'s own outcome loop gives every
3416    /// candidate) so the next agent starts clean.
3417    ///
3418    /// The new agent gets the implementer's full prompt and full
3419    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3420    /// it has no session and no context, and is implementing the task from
3421    /// nothing, unlike a resumed drop which is only restating work already
3422    /// done.
3423    ///
3424    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3425    /// `implement` event, never into `self.state.quota` — that is what
3426    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3427    /// go unspent, and a seat that ultimately recovered on its second or
3428    /// third agent is not the stalled panel that check exists to catch. Only
3429    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3430    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3431    /// outcome loop already has — this helper never pushes to it itself.
3432    async fn resume_seat_handovers(
3433        &mut self,
3434        results: &mut [(usize, SeatState, AgentOutcome)],
3435        sent: &mut [SeatJob],
3436        prompts: &Prompts,
3437        run_id: &str,
3438    ) -> Result<()> {
3439        let instruction = seeded_instruction(&self.state);
3440        let language = self.state.config.graph.language.clone();
3441        let github_language = self.state.config.graph.github_language.clone();
3442        let brief = self
3443            .state
3444            .advice
3445            .as_ref()
3446            .and_then(|a| a.synthesis.as_deref())
3447            .map(str::to_owned);
3448        let attachments = self.state.attachments.clone();
3449        for (wi, seat, out) in results.iter_mut() {
3450            // Who holds the other candidate seats of this wave right now
3451            // (earlier handovers already written back to `sent`).
3452            let others: BTreeSet<String> = sent
3453                .iter()
3454                .enumerate()
3455                .filter(|(j, _)| j != wi)
3456                .map(|(_, j)| j.spec.id.clone())
3457                .collect();
3458            let Some(job) = sent.get_mut(*wi) else {
3459                continue;
3460            };
3461            // Where the seat's own original agent sits in the roster — the
3462            // fallback walk starts just past here, never at the front, so a
3463            // later candidate slot's quota loss does not fall back onto an
3464            // earlier slot's own agent.
3465            let start = self
3466                .roles
3467                .implementer_roster
3468                .iter()
3469                .position(|s| s.id == job.spec.id)
3470                .unwrap_or(0);
3471            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3472            let mut fallback_attempt = 0usize;
3473            let mut prev: Option<FailClass> = None;
3474            while let Some(cur) = FailClass::of(&*out) {
3475                if !should_hand_over(prev.as_ref(), &cur) {
3476                    break;
3477                }
3478                let Some(next) =
3479                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3480                        .cloned()
3481                else {
3482                    break;
3483                };
3484                tried.insert(next.id.clone());
3485                fallback_attempt += 1;
3486
3487                let coordination = self.coordinate(&job.cwd).await?;
3488                if let Ok(r) = git::rescue_commit(
3489                    &job.cwd,
3490                    &format!(
3491                        "magi: candidate {} (uncommitted work before {} fallback)",
3492                        seat.key,
3493                        if cur == FailClass::Quota {
3494                            "quota"
3495                        } else {
3496                            "handover"
3497                        }
3498                    ),
3499                )
3500                .await
3501                {
3502                    self.state.note_withheld("implement", &r.withheld);
3503                }
3504                drop(coordination);
3505
3506                record_handover(
3507                    &mut self.state,
3508                    "implement",
3509                    &seat.key,
3510                    &seat.agent,
3511                    &next.id,
3512                    &cur,
3513                    &fail_reason(&*out),
3514                );
3515                prev = Some(cur.clone());
3516
3517                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3518                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3519                // Kept in sync on `sent` itself, not just the local retry: a
3520                // later helper (`resume_unconfirmed_commands`) reads `sent`
3521                // after this one returns and must see whichever agent is now
3522                // occupying the seat, not the one that just quota'd out —
3523                // otherwise it would judge session/continuation eligibility
3524                // by the wrong CLI and could resend a fallback's session id
3525                // to the agent that lost it the seat in the first place.
3526                job.spec = next.clone();
3527                let mut retry = job.clone();
3528                retry.seat = new_seat;
3529                retry.prompt = prompt::implement_in(
3530                    &instruction,
3531                    &job.cwd.to_string_lossy(),
3532                    &language,
3533                    &github_language,
3534                    brief.as_deref(),
3535                    &attachments,
3536                );
3537                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3538                let cache = self.state.config.cache_dir();
3539                let ctx = WaveCtx {
3540                    carry_seats: false,
3541                    pauses: Some((&self.pause, &self.interrupt)),
3542                    run: run_id,
3543                    node: "implement",
3544                    prompts,
3545                    cache: cache.as_deref(),
3546                    round: None,
3547                };
3548                let (fallback_seat, fallback_out) = run_one(
3549                    retry,
3550                    Arc::clone(&self.sem),
3551                    &ctx,
3552                    &mut self.state,
3553                    fallback_attempt,
3554                )
3555                .await;
3556                *seat = fallback_seat;
3557                *out = fallback_out;
3558            }
3559        }
3560        Ok(())
3561    }
3562
3563    /// Ask an implement seat's own CLI to confirm what it started, once, when
3564    /// its reply reported a command whose completion status it never
3565    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3566    /// that does and does not mean.
3567    ///
3568    /// The completion contract this task asks for, extended to `implement`
3569    /// with the same signal `continue_fix_report` reads for the fixer,
3570    /// rather than a keyword search over the reply or a hard requirement on
3571    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3572    /// a candidate's CLI turn ended cleanly while a test run it had started
3573    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3574    /// named in it at all is untouched by this: `commands` is empty, so
3575    /// there is nothing to be unconfirmed.
3576    ///
3577    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3578    /// this is not about recovering edits that might already be on disk, it
3579    /// is about a result the seat itself never vouched for, which resuming
3580    /// asks for regardless of what the tree already holds. Bounded to one
3581    /// attempt for the same reason `resume_undelivered` is — this is the
3582    /// most expensive node in the graph — and a seat that still cannot
3583    /// confirm on that attempt is left as whatever its (possibly still
3584    /// unconfirmed) reply says; this does not invent a new "failed" reason
3585    /// for a candidate that otherwise produced a real, committed change.
3586    async fn resume_unconfirmed_commands(
3587        &mut self,
3588        results: &mut [(usize, SeatState, AgentOutcome)],
3589        sent: &[SeatJob],
3590        prompts: &Prompts,
3591        run_id: &str,
3592    ) {
3593        for (wi, seat, out) in results.iter_mut() {
3594            let AgentOutcome::Ok(o) = &*out else {
3595                continue;
3596            };
3597            if !has_unconfirmed_command(&o.commands) {
3598                continue;
3599            }
3600            let Some(job) = sent.get(*wi) else { continue };
3601            if !has_context(&job.spec, seat, job.sessions) {
3602                self.state.event(
3603                    "implement",
3604                    format!(
3605                        "{}: the reply named a command whose own CLI never confirmed the exit \
3606                         status of, but there is no session left to resume",
3607                        seat.key
3608                    ),
3609                );
3610                continue;
3611            }
3612            self.state.event(
3613                "implement",
3614                format!(
3615                    "{}: the reply named a command whose own CLI never confirmed the exit \
3616                     status of; resuming the conversation",
3617                    seat.key
3618                ),
3619            );
3620            let mut retry = job.clone();
3621            retry.seat = seat.clone();
3622            retry.prompt = prompt::resume_incomplete(
3623                "a command in your last reply had no confirmed exit status",
3624            );
3625            retry.timeout = retry_budget(job.timeout, true);
3626            retry.stem = format!("{}-confirm", job.stem);
3627            let cache = self.state.config.cache_dir();
3628            let ctx = WaveCtx {
3629                carry_seats: false,
3630                pauses: Some((&self.pause, &self.interrupt)),
3631                run: run_id,
3632                node: "implement",
3633                prompts,
3634                cache: cache.as_deref(),
3635                round: None,
3636            };
3637            let (resumed_seat, resumed) =
3638                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3639            *seat = resumed_seat;
3640            *out = resumed;
3641        }
3642    }
3643
3644    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3645    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3646    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3647    /// that motivated this.
3648    ///
3649    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3650    /// own nudge loop already covers for judge/review/vote seats, and not a
3651    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3652    /// implement seats: here the CLI turn genuinely finished while the node's
3653    /// own work — the fixer's account of what it did — had not. Gated purely
3654    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3655    /// reply, never on any wording in it, so a fixer whose valid, first-try
3656    /// `FixReport` happens to mention having waited on a background test is
3657    /// never resumed — the `Ok(report)` branch at the call site returns
3658    /// before this is ever invoked.
3659    ///
3660    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3661    /// attempt ([`retry_budget`]), nothing attempted once the session is
3662    /// gone, and a quota hit ends the loop immediately rather than retrying a
3663    /// rate limit that fails the same way again.
3664    async fn continue_fix_report(
3665        &mut self,
3666        mut seat: SeatState,
3667        parse_err: String,
3668        job: &SeatJob,
3669        prompts: &Prompts,
3670        run_id: &str,
3671        round: usize,
3672    ) -> (
3673        SeatState,
3674        Option<FixReport>,
3675        Option<String>,
3676        ContinuationRecord,
3677    ) {
3678        let mut last_err = parse_err;
3679        let mut cumulative_wait_ms = 0u64;
3680        let mut attempts = 0usize;
3681        loop {
3682            if !has_context(&job.spec, &seat, job.sessions) {
3683                self.state.event(
3684                    "fix",
3685                    format!(
3686                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3687                         session left to resume into"
3688                    ),
3689                );
3690                let outcome = if attempts == 0 {
3691                    ContinuationOutcome::NoSession
3692                } else {
3693                    ContinuationOutcome::Exhausted
3694                };
3695                return (
3696                    seat,
3697                    None,
3698                    Some(format!("unparsable fix report: {last_err}")),
3699                    ContinuationRecord {
3700                        attempts,
3701                        cumulative_wait_ms,
3702                        outcome,
3703                    },
3704                );
3705            }
3706            if attempts >= MAX_FIX_CONTINUATIONS {
3707                self.state.event(
3708                    "fix",
3709                    format!(
3710                        "round {round}: fixer's reply still had no adoption report after \
3711                         {attempts} continuation(s) ({last_err}); giving up"
3712                    ),
3713                );
3714                return (
3715                    seat,
3716                    None,
3717                    Some(format!(
3718                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3719                    )),
3720                    ContinuationRecord {
3721                        attempts,
3722                        cumulative_wait_ms,
3723                        outcome: ContinuationOutcome::Exhausted,
3724                    },
3725                );
3726            }
3727            attempts += 1;
3728            self.state.event(
3729                "fix",
3730                format!(
3731                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3732                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3733                ),
3734            );
3735            let mut retry = job.clone();
3736            retry.seat = seat.clone();
3737            retry.prompt = prompt::resume_incomplete(&last_err);
3738            retry.timeout = retry_budget(job.timeout, true);
3739            retry.stem = format!("{}-continue{attempts}", job.stem);
3740            let cache = self.state.config.cache_dir();
3741            let ctx = WaveCtx {
3742                carry_seats: false,
3743                pauses: Some((&self.pause, &self.interrupt)),
3744                run: run_id,
3745                node: "fix",
3746                prompts,
3747                cache: cache.as_deref(),
3748                round: Some(round),
3749            };
3750            let (resumed_seat, resumed_out) = run_one(
3751                retry,
3752                Arc::clone(&self.sem),
3753                &ctx,
3754                &mut self.state,
3755                attempts,
3756            )
3757            .await;
3758            seat = resumed_seat;
3759            match resumed_out {
3760                AgentOutcome::Ok(o) => {
3761                    cumulative_wait_ms += o.duration_ms;
3762                    match verdict::extract_json::<FixReport>(&o.text) {
3763                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3764                            self.state.event(
3765                                "fix",
3766                                format!(
3767                                    "round {round}: fixer's adoption report recovered after \
3768                                     {attempts} continuation(s)"
3769                                ),
3770                            );
3771                            return (
3772                                seat,
3773                                Some(report),
3774                                None,
3775                                ContinuationRecord {
3776                                    attempts,
3777                                    cumulative_wait_ms,
3778                                    outcome: ContinuationOutcome::Resumed,
3779                                },
3780                            );
3781                        }
3782                        // The report parsed, but this same reply's own
3783                        // CommandEvidence — the identical record `state.jobs`
3784                        // renders — names a command whose CLI never
3785                        // confirmed an exit status. Read together, that is
3786                        // not a resolved answer: keep nudging rather than
3787                        // accept a report standing next to a command the
3788                        // seat's own CLI cannot vouch for.
3789                        Ok(_) => {
3790                            last_err = "the reply parsed, but it reported a command whose own CLI \
3791                                 never confirmed an exit status"
3792                                .to_owned();
3793                        }
3794                        Err(e) => last_err = e.to_string(),
3795                    }
3796                }
3797                AgentOutcome::Quota(o) => {
3798                    cumulative_wait_ms += o.duration_ms;
3799                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3800                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3801                    self.state.quota.push(loss);
3802                    self.state.event(
3803                        "fix",
3804                        format!(
3805                            "round {round}: continuation rate limited (quota); not retrying now"
3806                        ),
3807                    );
3808                    return (
3809                        seat,
3810                        None,
3811                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3812                        ContinuationRecord {
3813                            attempts,
3814                            cumulative_wait_ms,
3815                            outcome: ContinuationOutcome::QuotaLost,
3816                        },
3817                    );
3818                }
3819                AgentOutcome::Dropped(o) => {
3820                    cumulative_wait_ms += o.duration_ms;
3821                    let why = o
3822                        .dropped
3823                        .as_ref()
3824                        .map(|d| d.why.as_str())
3825                        .unwrap_or("the CLI ended the stream without delivering its answer");
3826                    last_err = format!("the CLI dropped the stream ({why})");
3827                }
3828                AgentOutcome::Failed(e) => last_err = e,
3829                // The continuation was never started; the report stays lost
3830                // and the caller records that, without any handover.
3831                AgentOutcome::ResourceBlocked(e) => {
3832                    return (
3833                        seat,
3834                        None,
3835                        Some(format!("the continuation was not started: {e}")),
3836                        ContinuationRecord {
3837                            attempts: attempts.saturating_sub(1),
3838                            cumulative_wait_ms,
3839                            outcome: ContinuationOutcome::NoSession,
3840                        },
3841                    );
3842                }
3843            }
3844        }
3845    }
3846
3847    fn after_implement(&mut self) -> Result<()> {
3848        if !self.state.candidate_preservation.is_empty() {
3849            bail!("candidate preservation incomplete; judging is unsafe");
3850        }
3851        // Scan every candidate patch once the set is complete.
3852        if self.state.leaks.is_empty() {
3853            let cfg = self.state.config.blind.clone();
3854            let mut leaks = Vec::new();
3855            for c in &self.state.candidates {
3856                let Some(patch) =
3857                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3858                else {
3859                    continue;
3860                };
3861                leaks.extend(blind::scan(
3862                    &format!("candidate {} patch", c.label),
3863                    &patch,
3864                    &cfg.vendor_tokens,
3865                ));
3866            }
3867            if !leaks.is_empty() {
3868                let summary = leaks
3869                    .iter()
3870                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3871                    .collect::<Vec<_>>()
3872                    .join(", ");
3873                match cfg.on_leak {
3874                    LeakPolicy::Fail => {
3875                        self.state.status = RunStatus::Failed;
3876                        self.state
3877                            .event("blind", format!("vendor text in a patch: {summary}"));
3878                        self.state.leaks = leaks;
3879                        self.state.save()?;
3880                        self.settle_questions();
3881                        bail!(
3882                            "blind.on_leak = \"fail\" and vendor text reached a \
3883                             judged patch: {summary}"
3884                        );
3885                    }
3886                    LeakPolicy::Redact => self.state.event(
3887                        "blind",
3888                        format!("redacting vendor text for judging: {summary}"),
3889                    ),
3890                    LeakPolicy::Warn => self.state.event(
3891                        "blind",
3892                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3893                    ),
3894                }
3895                self.state.leaks = leaks;
3896            }
3897        }
3898
3899        if self.state.viable().is_empty() {
3900            if self.state.all_candidates_verified_noop() {
3901                // Every candidate agreed, with evidence the adoption guard
3902                // accepted, that nothing belongs in this worktree. That is
3903                // not the same fact as a candidate that simply failed to
3904                // write anything, and settling it as an ordinary `Failed`
3905                // (see `SCHEMA`'s doc for schema 10) is what let two of
3906                // task 391f's attempts burn a retry each re-discovering the
3907                // same already-landed fix. Terminal either way, so `judge`
3908                // must never run over an empty candidate set — unlike the
3909                // `Failed` branch below this returns `Ok`, not an error:
3910                // nothing here failed.
3911                self.state.status = RunStatus::VerifiedNoop;
3912                self.state.save()?;
3913                self.settle_questions();
3914                return Ok(());
3915            }
3916            self.state.status = RunStatus::Failed;
3917            self.state.save()?;
3918            self.settle_questions();
3919            bail!("no candidate produced a change; nothing to judge");
3920        }
3921        self.state.status = RunStatus::Judging;
3922        self.state.save()?;
3923        Ok(())
3924    }
3925
3926    // --------------------------------------------------------------- judge
3927
3928    async fn judge(&mut self) -> Result<()> {
3929        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3930        // agent files with `magi task add` name the run that paid for it. The
3931        // prompt overlay is cloned alongside it because the waves borrow it
3932        // while `self` is mutably borrowed by the node's own bookkeeping.
3933        let run_id = self.state.id.clone();
3934        let prompts = self.state.config.prompts.clone();
3935        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3936            return Ok(());
3937        }
3938        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3939        if viable.len() == 1 {
3940            // Recorded so this is a one-time event: `judgements` stays empty
3941            // either way, which without this flag is indistinguishable from
3942            // "not yet judged" on the next reentry — and status is left
3943            // untouched, so a later node's conclusion (e.g. `Blocked` after
3944            // the review budget ran out) survives a resume instead of being
3945            // clobbered back to `Judging` by this node running again.
3946            self.state.judge_skipped = true;
3947            self.state.event(
3948                "judge",
3949                format!(
3950                    "only candidate {} produced a change; judging skipped",
3951                    viable[0].label
3952                ),
3953            );
3954            self.state.save()?;
3955            return Ok(());
3956        }
3957        self.state.status = RunStatus::Judging;
3958
3959        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3960        let language = self.state.config.graph.language.clone();
3961        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3962        let sessions = self.state.config.graph.sessions;
3963        let artifacts = agent::artifacts_dir(&self.state.dir());
3964        let root = self.state.worktree_root();
3965        let base_short = short(&self.state.base_commit);
3966
3967        let mut jobs = Vec::new();
3968        let mut orders = Vec::new();
3969        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3970            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3971            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3972            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3973            let seat_key = format!("judge-{}", j + 1);
3974            let seat = self.seat(&seat_key, &spec.id);
3975            jobs.push(SeatJob {
3976                prompt: prompt::judge(
3977                    &self.state.instruction,
3978                    &views,
3979                    self.roles.judges.len(),
3980                    &base_short,
3981                    &language,
3982                ),
3983                spec,
3984                seat,
3985                cwd: root.join(format!("judge-{}", j + 1)),
3986                timeout,
3987                allow_write: false,
3988                sessions,
3989                artifacts: artifacts.clone(),
3990                stem: format!("judge-{}", j + 1),
3991                handover: None,
3992            });
3993        }
3994
3995        self.state.event(
3996            "judge",
3997            format!(
3998                "{} judges ranking {} candidates blind",
3999                jobs.len(),
4000                viable.len()
4001            ),
4002        );
4003        let labels_for_check = labels.clone();
4004        let mut quota_losses = Vec::new();
4005        let cache = self.state.config.cache_dir();
4006        let ctx = WaveCtx {
4007            carry_seats: false,
4008            pauses: Some((&self.pause, &self.interrupt)),
4009            run: &run_id,
4010            node: "judge",
4011            prompts: &prompts,
4012            cache: cache.as_deref(),
4013            round: None,
4014        };
4015        let results = ask_json_wave::<Ranking>(
4016            jobs,
4017            Arc::clone(&self.sem),
4018            self.state.config.graph.retries,
4019            &self.roles.judge_roster,
4020            &ctx,
4021            &mut quota_losses,
4022            &mut self.state,
4023            &move |r: &Ranking| r.validate(&labels_for_check),
4024        )
4025        .await;
4026        if self.state.resource_block.is_some() {
4027            return self.stop_for_resource();
4028        }
4029        self.state.quota.extend(quota_losses);
4030
4031        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4032            let agent_id = seat.agent.clone();
4033            self.state.seats.insert(seat.key.clone(), seat);
4034            let mut record = Judgement {
4035                judge: j + 1,
4036                seat: format!("judge-{}", j + 1),
4037                agent: agent_id,
4038                ranking: Vec::new(),
4039                reasons: BTreeMap::new(),
4040                confidence: None,
4041                strengths: BTreeMap::new(),
4042                order: orders[j].clone(),
4043                failed: None,
4044                duration_ms: 0,
4045            };
4046            match res {
4047                Ok((ranking, out)) => {
4048                    record.ranking = ranking.normalized();
4049                    record.reasons = ranking.reasons;
4050                    record.strengths = ranking.strengths;
4051                    record.confidence = ranking.confidence;
4052                    record.duration_ms = out.duration_ms;
4053                    self.state.event(
4054                        "judge",
4055                        format!(
4056                            "judge {} ranked {}",
4057                            j + 1,
4058                            record.ranking.iter().collect::<String>()
4059                        ),
4060                    );
4061                }
4062                Err(e) => {
4063                    record.failed = Some(e.to_string());
4064                    self.state
4065                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4066                }
4067            }
4068            self.state.judgements.push(record);
4069            self.state.save()?;
4070        }
4071        Ok(())
4072    }
4073
4074    // ---------------------------------------------------------- deliberate
4075
4076    async fn deliberate(&mut self) -> Result<()> {
4077        if self.hold_base_sync().await? {
4078            return Ok(());
4079        }
4080        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4081        // agent files with `magi task add` name the run that paid for it. The
4082        // prompt overlay is cloned alongside it because the waves borrow it
4083        // while `self` is mutably borrowed by the node's own bookkeeping.
4084        let run_id = self.state.id.clone();
4085        let prompts = self.state.config.prompts.clone();
4086        let rounds = self.state.config.graph.deliberate_rounds;
4087        // Resume unfinished rounds after an outage, but never reopen a tally.
4088        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4089            return Ok(());
4090        }
4091        let tops: Vec<char> = self
4092            .state
4093            .judgements
4094            .iter()
4095            .filter_map(|j| j.ranking.first().copied())
4096            .collect();
4097        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4098            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4099                self.state.event(
4100                    "deliberate",
4101                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4102                );
4103            }
4104            self.state.status = RunStatus::Voting;
4105            self.state.save()?;
4106            return Ok(());
4107        }
4108
4109        self.state.status = RunStatus::Deliberating;
4110        self.state.event(
4111            "deliberate",
4112            format!(
4113                "split: first choices were {} — opening {rounds} round(s)",
4114                tops.iter().collect::<String>()
4115            ),
4116        );
4117
4118        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4119        let language = self.state.config.graph.language.clone();
4120        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4121        let sessions = self.state.config.graph.sessions;
4122        let artifacts = agent::artifacts_dir(&self.state.dir());
4123        let root = self.state.worktree_root();
4124        let base_short = short(&self.state.base_commit);
4125
4126        // Judges argue in sequence so that a turn can answer the one before it;
4127        // that is the difference between deliberation and three parallel
4128        // monologues.
4129        for round in self.state.deliberation.len() + 1..=rounds {
4130            let mut turns: Vec<DeliberationTurn> = Vec::new();
4131            let mut round_losses = Vec::new();
4132            let mut asked = 0;
4133            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4134                if self.state.judgements[j].failed.is_some() {
4135                    continue;
4136                }
4137                let seat_key = format!("judge-{}", j + 1);
4138                let spec = self.occupant(&seat_key, spec);
4139                let mut seat = self.seat(&seat_key, &spec.id);
4140                let transcript = self.transcript(&turns, j);
4141                let build = |context: Option<&str>| {
4142                    prompt::deliberate(
4143                        &self.state.instruction,
4144                        context,
4145                        &transcript,
4146                        round,
4147                        rounds,
4148                        &language,
4149                    )
4150                };
4151                let block = self.candidate_block(&viable, &base_short);
4152                let full = build(Some(&block));
4153                let text = if has_context(&spec, &seat, sessions) {
4154                    build(None)
4155                } else {
4156                    full.clone()
4157                };
4158                let job = SeatJob {
4159                    spec,
4160                    seat: seat.clone(),
4161                    prompt: text,
4162                    cwd: root.join(format!("judge-{}", j + 1)),
4163                    timeout,
4164                    allow_write: false,
4165                    sessions,
4166                    artifacts: artifacts.clone(),
4167                    stem: format!("delib-{round}-judge-{}", j + 1),
4168                    handover: Some(full),
4169                };
4170                let cache = self.state.config.cache_dir();
4171                let ctx = WaveCtx {
4172                    carry_seats: false,
4173                    pauses: Some((&self.pause, &self.interrupt)),
4174                    run: &run_id,
4175                    node: "deliberate",
4176                    prompts: &prompts,
4177                    cache: cache.as_deref(),
4178                    round: None,
4179                };
4180                // A turn is never nudged (`retries` 0): a failed seat is
4181                // handed to the next roster agent, which gets the full
4182                // context. An empty answer is a turn, not a failure.
4183                let mut losses = Vec::new();
4184                asked += 1;
4185                let mut results = ask_wave_with::<String>(
4186                    vec![job],
4187                    Arc::clone(&self.sem),
4188                    0,
4189                    &self.roles.judge_roster,
4190                    &ctx,
4191                    &mut losses,
4192                    &mut self.state,
4193                    &|text: &str| {
4194                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4195                    },
4196                )
4197                .await;
4198                if self.state.resource_block.is_some() {
4199                    return self.stop_for_resource();
4200                }
4201                round_losses.extend(losses.iter().cloned());
4202                self.state.quota.extend(losses);
4203                let (updated, res, _) = results.pop().expect("one job in, one result out");
4204                seat = updated;
4205                let agent_id = seat.agent.clone();
4206                self.state.seats.insert(seat.key.clone(), seat);
4207                let body = match res {
4208                    Ok((body, _)) => body,
4209                    // Skip the seat; a CLI's raw error JSON is never read as
4210                    // this judge's position.
4211                    Err(e) => {
4212                        self.state
4213                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4214                        continue;
4215                    }
4216                };
4217                let tentative = verdict::extract_json::<Position>(&body)
4218                    .ok()
4219                    .and_then(|p| p.tentative)
4220                    .and_then(|s| s.trim().chars().next())
4221                    .map(|c| c.to_ascii_uppercase());
4222                self.state.event(
4223                    "deliberate",
4224                    format!(
4225                        "round {round}: judge {} now favours {}",
4226                        j + 1,
4227                        tentative.map_or("—".to_owned(), |c| c.to_string())
4228                    ),
4229                );
4230                turns.push(DeliberationTurn {
4231                    judge: j + 1,
4232                    agent: agent_id,
4233                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4234                    tentative,
4235                });
4236            }
4237            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4238                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4239                return self.stop_for_resource();
4240            }
4241            self.state
4242                .deliberation
4243                .push(DeliberationRound { round, turns });
4244            self.state.save()?;
4245        }
4246
4247        self.state.status = RunStatus::Voting;
4248        self.state.save()?;
4249        Ok(())
4250    }
4251
4252    // ---------------------------------------------------------------- vote
4253
4254    async fn vote(&mut self) -> Result<()> {
4255        if self.hold_base_sync().await? {
4256            return Ok(());
4257        }
4258        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4259        // agent files with `magi task add` name the run that paid for it. The
4260        // prompt overlay is cloned alongside it because the waves borrow it
4261        // while `self` is mutably borrowed by the node's own bookkeeping.
4262        let run_id = self.state.id.clone();
4263        let prompts = self.state.config.prompts.clone();
4264        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4265            return Ok(());
4266        }
4267        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4268        if viable.len() == 1 {
4269            return Ok(());
4270        }
4271        self.state.status = RunStatus::Voting;
4272
4273        let language = self.state.config.graph.language.clone();
4274        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4275        let sessions = self.state.config.graph.sessions;
4276        let artifacts = agent::artifacts_dir(&self.state.dir());
4277        let root = self.state.worktree_root();
4278        let base_short = short(&self.state.base_commit);
4279        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4280
4281        let mut jobs = Vec::new();
4282        let mut seats_at = Vec::new();
4283        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4284            if self
4285                .state
4286                .judgements
4287                .get(j)
4288                .is_some_and(|r| r.failed.is_some())
4289            {
4290                continue;
4291            }
4292            let seat_key = format!("judge-{}", j + 1);
4293            let spec = self.occupant(&seat_key, spec);
4294            let seat = self.seat(&seat_key, &spec.id);
4295            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4296            let text = if has_context(&spec, &seat, sessions) {
4297                prompt::final_vote(&viable, &language)
4298            } else {
4299                full.clone()
4300            };
4301            jobs.push(SeatJob {
4302                spec,
4303                seat,
4304                prompt: text,
4305                cwd: root.join(format!("judge-{}", j + 1)),
4306                timeout,
4307                allow_write: false,
4308                sessions,
4309                artifacts: artifacts.clone(),
4310                stem: format!("vote-judge-{}", j + 1),
4311                handover: Some(full),
4312            });
4313            seats_at.push(j);
4314        }
4315
4316        self.state.event(
4317            "vote",
4318            format!(
4319                "collecting {} final votes one by one, privately",
4320                jobs.len()
4321            ),
4322        );
4323        let allowed = viable.clone();
4324        let mut quota_losses = Vec::new();
4325        let cache = self.state.config.cache_dir();
4326        let ctx = WaveCtx {
4327            carry_seats: false,
4328            pauses: Some((&self.pause, &self.interrupt)),
4329            run: &run_id,
4330            node: "vote",
4331            prompts: &prompts,
4332            cache: cache.as_deref(),
4333            round: None,
4334        };
4335        let results = ask_json_wave::<FinalVote>(
4336            jobs,
4337            Arc::clone(&self.sem),
4338            self.state.config.graph.retries,
4339            &self.roles.judge_roster,
4340            &ctx,
4341            &mut quota_losses,
4342            &mut self.state,
4343            &move |v: &FinalVote| match v.label() {
4344                Some(c) if allowed.contains(&c) => Ok(()),
4345                other => bail!("vote {other:?} is not one of {allowed:?}"),
4346            },
4347        )
4348        .await;
4349        if self.state.resource_block.is_some() {
4350            return self.stop_for_resource();
4351        }
4352        self.state.quota.extend(quota_losses);
4353
4354        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4355            let agent_id = seat.agent.clone();
4356            self.state.seats.insert(seat.key.clone(), seat);
4357            let initial = self
4358                .state
4359                .judgements
4360                .get(j)
4361                .and_then(|r| r.ranking.first().copied());
4362            let mut record = VoteRecord {
4363                judge: j + 1,
4364                agent: agent_id,
4365                vote: None,
4366                reason: String::new(),
4367                changed: false,
4368            };
4369            match res {
4370                Ok((v, _)) => {
4371                    record.vote = v.label();
4372                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4373                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4374                    self.state.event(
4375                        "vote",
4376                        format!(
4377                            "judge {} voted {}{}",
4378                            j + 1,
4379                            record.vote.unwrap_or('?'),
4380                            if record.changed { " (changed)" } else { "" }
4381                        ),
4382                    );
4383                }
4384                Err(e) => {
4385                    self.state
4386                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4387                }
4388            }
4389            self.state.votes.push(record);
4390            self.state.save()?;
4391        }
4392        Ok(())
4393    }
4394
4395    // --------------------------------------------------------------- tally
4396
4397    fn tally(&mut self) -> Result<()> {
4398        if self.state.tally.is_some() {
4399            return Ok(());
4400        }
4401        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4402        let tops: Vec<char> = self
4403            .state
4404            .judgements
4405            .iter()
4406            .filter_map(|j| j.ranking.first().copied())
4407            .collect();
4408        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4409
4410        // A judge whose private vote failed still counted once, in the initial
4411        // ranking; using it beats discarding a whole seat.
4412        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4413        let mut cast: Vec<char> = Vec::new();
4414        for (i, j) in self.state.judgements.iter().enumerate() {
4415            let vote = self
4416                .state
4417                .votes
4418                .iter()
4419                .find(|v| v.judge == i + 1)
4420                .and_then(|v| v.vote)
4421                .or_else(|| j.ranking.first().copied());
4422            if let Some(v) = vote {
4423                *first_choice.entry(v).or_insert(0) += 1;
4424                cast.push(v);
4425            }
4426        }
4427
4428        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4429        for j in &self.state.judgements {
4430            let n = j.ranking.len();
4431            for (pos, label) in j.ranking.iter().enumerate() {
4432                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4433            }
4434        }
4435
4436        let best = first_choice.values().copied().max().unwrap_or(0);
4437        let mut leaders: Vec<char> = first_choice
4438            .iter()
4439            .filter(|(_, v)| **v == best)
4440            .map(|(k, _)| *k)
4441            .collect();
4442        let mut tie_break = None;
4443        if leaders.len() > 1 {
4444            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4445            let borda_leaders: Vec<char> = leaders
4446                .iter()
4447                .copied()
4448                .filter(|l| borda[l] == top_borda)
4449                .collect();
4450            tie_break = Some(if borda_leaders.len() == 1 {
4451                format!(
4452                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4453                    leaders.len()
4454                )
4455            } else {
4456                format!(
4457                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4458                    leaders.len()
4459                )
4460            });
4461            leaders = borda_leaders;
4462            leaders.sort_unstable();
4463        }
4464        let winner = *leaders
4465            .first()
4466            .or(viable.first())
4467            .context("no candidate to declare a winner from")?;
4468
4469        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4470        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4471        let deliberated = !self.state.deliberation.is_empty();
4472
4473        // Whose verdict is this? A rate-limited seat is absent even if it
4474        // ranked before the limit hit, so presence is measured against the
4475        // recorded losses, not just "did a ranking ever appear".
4476        let quota_seats: std::collections::BTreeSet<&str> =
4477            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4478        let mut present = 0usize;
4479        for (i, j) in self.state.judgements.iter().enumerate() {
4480            if quota_seats.contains(j.seat.as_str()) {
4481                continue;
4482            }
4483            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4484            let voted = self
4485                .state
4486                .votes
4487                .iter()
4488                .any(|v| v.judge == i + 1 && v.vote.is_some());
4489            if ranked || voted {
4490                present += 1;
4491            }
4492        }
4493        // Strict majority of the configured panel. A bare majority is real
4494        // signal we can act on, while a minority verdict must never stand in
4495        // for a healthy one. A one-candidate run needs no panel at all, and
4496        // `judges` stays `0` rather than the roster size a panel that never
4497        // sat would otherwise be credited with.
4498        let needs_quorum = viable.len() > 1;
4499        let judges_total = if needs_quorum {
4500            self.roles.judges.len()
4501        } else {
4502            0
4503        };
4504        let quorum = if needs_quorum {
4505            judges_total / 2 + 1
4506        } else {
4507            0
4508        };
4509        let met_quorum = !needs_quorum || present >= quorum;
4510        let uncontested = (!needs_quorum).then(|| {
4511            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4512        });
4513
4514        self.state.event(
4515            "tally",
4516            match &uncontested {
4517                Some(reason) => format!("winner {winner} — {reason}"),
4518                None => format!(
4519                    "winner {winner} — votes {} | initial {} | {} changed | \
4520                     {present}/{judges_total} judges{}",
4521                    first_choice
4522                        .iter()
4523                        .map(|(k, v)| format!("{k}:{v}"))
4524                        .collect::<Vec<_>>()
4525                        .join(" "),
4526                    if unanimous_initial {
4527                        "unanimous"
4528                    } else {
4529                        "split"
4530                    },
4531                    changed_votes,
4532                    if met_quorum {
4533                        String::new()
4534                    } else {
4535                        format!(" — below quorum ({quorum} required)")
4536                    },
4537                ),
4538            },
4539        );
4540        if !met_quorum {
4541            self.state.event(
4542                "stall",
4543                format!(
4544                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4545                     the run stops here, resumable"
4546                ),
4547            );
4548        }
4549        self.state.tally = Some(Tally {
4550            first_choice,
4551            borda,
4552            winner,
4553            rankings: tops.len(),
4554            unanimous_initial,
4555            deliberated,
4556            changed_votes,
4557            unanimous_final,
4558            tie_break,
4559            judges: judges_total,
4560            present,
4561            quorum,
4562            met_quorum,
4563            uncontested,
4564        });
4565        self.state.status = if met_quorum {
4566            RunStatus::Reviewing
4567        } else {
4568            RunStatus::Stalled
4569        };
4570        self.state.save()?;
4571        Ok(())
4572    }
4573
4574    // ------------------------------------------------------------- recover
4575
4576    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4577    /// resumed toward completion once the transient cause clears.
4578    ///
4579    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4580    /// it toward the quorum, which is exactly the set of seats whose absence
4581    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4582    /// not depend on which node happened to hit the limit), or an ordinary
4583    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4584    /// seat is never disturbed.
4585    ///
4586    /// A seat that now answers with a usable ranking is "recovered": its
4587    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4588    /// `tally` counts it present again), and its vote re-collected. A seat that
4589    /// still fails keeps its loss and stays absent.
4590    ///
4591    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4592    /// to review/gate/merge), `false` when it is still below quorum (the run
4593    /// stays `Stalled`, still resumable for a later retry).
4594    #[allow(clippy::too_many_lines)]
4595    async fn recover_stall(&mut self) -> Result<bool> {
4596        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4597        // agent files with `magi task add` name the run that paid for it. The
4598        // prompt overlay is cloned alongside it because the waves borrow it
4599        // while `self` is mutably borrowed by the node's own bookkeeping.
4600        let run_id = self.state.id.clone();
4601        let prompts = self.state.config.prompts.clone();
4602        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4603        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4604        // is what keeps a non-quota collapse — or a quota loss recorded at the
4605        // deliberate node — from being a permanent dead-end on `--resume`.
4606        let quota_seats: BTreeSet<&str> =
4607            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4608        let absent: Vec<String> = self
4609            .state
4610            .judgements
4611            .iter()
4612            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4613            .map(|j| j.seat.clone())
4614            .collect();
4615        if absent.is_empty() {
4616            return Ok(false);
4617        }
4618        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4619        if viable.len() <= 1 {
4620            return Ok(false);
4621        }
4622        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4623        let language = self.state.config.graph.language.clone();
4624        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4625        let sessions = self.state.config.graph.sessions;
4626        let artifacts = agent::artifacts_dir(&self.state.dir());
4627        let root = self.state.worktree_root();
4628        let base_short = short(&self.state.base_commit);
4629        let candidates: Vec<Candidate> = viable.clone();
4630
4631        // Map each absent seat key to its 0-based position in `roles.judges`.
4632        let mut positions: Vec<usize> = absent
4633            .iter()
4634            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4635            .collect();
4636        if positions.is_empty() {
4637            return Ok(false);
4638        }
4639        positions.sort_unstable();
4640        positions.dedup();
4641
4642        // Re-rank the lost seats, one blind prompt each.
4643        let mut judge_jobs = Vec::new();
4644        for &j in &positions {
4645            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4646            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4647            let seat_key = format!("judge-{}", j + 1);
4648            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4649            let seat = self.seat(&seat_key, &spec.id);
4650            judge_jobs.push(SeatJob {
4651                spec,
4652                seat,
4653                prompt: prompt::judge(
4654                    &self.state.instruction,
4655                    &views,
4656                    self.roles.judges.len(),
4657                    &base_short,
4658                    &language,
4659                ),
4660                cwd: root.join(seat_key),
4661                timeout,
4662                allow_write: false,
4663                sessions,
4664                artifacts: artifacts.clone(),
4665                stem: format!("judge-{}-recover", j + 1),
4666                handover: None,
4667            });
4668        }
4669
4670        let labels_for_check = labels.clone();
4671        let mut judge_losses = Vec::new();
4672        let retries = self.state.config.graph.retries;
4673        let cache = self.state.config.cache_dir();
4674        let ctx = WaveCtx {
4675            carry_seats: false,
4676            pauses: Some((&self.pause, &self.interrupt)),
4677            run: &run_id,
4678            node: "judge",
4679            prompts: &prompts,
4680            cache: cache.as_deref(),
4681            round: None,
4682        };
4683        let results = ask_json_wave::<Ranking>(
4684            judge_jobs,
4685            Arc::clone(&self.sem),
4686            retries,
4687            &self.roles.judge_roster,
4688            &ctx,
4689            &mut judge_losses,
4690            &mut self.state,
4691            &move |r: &Ranking| r.validate(&labels_for_check),
4692        )
4693        .await;
4694        if self.state.resource_block.is_some() {
4695            self.stop_for_resource()?;
4696            return Ok(false);
4697        }
4698
4699        // Refresh the judgement of every seat that ranked again.
4700        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4701        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4702            let agent_id = seat.agent.clone();
4703            self.state.seats.insert(seat.key.clone(), seat);
4704            let record = &mut self.state.judgements[j];
4705            match res {
4706                Ok((ranking, out)) => {
4707                    record.agent = agent_id;
4708                    record.ranking = ranking.normalized();
4709                    record.reasons = ranking.reasons;
4710                    record.strengths = ranking.strengths;
4711                    record.confidence = ranking.confidence;
4712                    record.failed = None;
4713                    record.duration_ms = out.duration_ms;
4714                    recovered.insert(j);
4715                    self.state.event(
4716                        "recover",
4717                        format!("judge {} ranked again after the limit", j + 1),
4718                    );
4719                }
4720                Err(e) => {
4721                    self.state
4722                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4723                }
4724            }
4725        }
4726
4727        // Re-ask the votes of the seats that recovered a ranking.
4728        let mut vote_jobs = Vec::new();
4729        let mut vote_pos: Vec<usize> = Vec::new();
4730        for &j in &recovered {
4731            let seat_key = format!("judge-{}", j + 1);
4732            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4733            let seat = self.seat(&seat_key, &spec.id);
4734            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4735            let text = if has_context(&spec, &seat, sessions) {
4736                prompt::final_vote(&labels, &language)
4737            } else {
4738                full.clone()
4739            };
4740            vote_jobs.push(SeatJob {
4741                spec,
4742                seat,
4743                prompt: text,
4744                cwd: root.join(seat_key),
4745                timeout,
4746                allow_write: false,
4747                sessions,
4748                artifacts: artifacts.clone(),
4749                stem: format!("vote-judge-{}-recover", j + 1),
4750                handover: Some(full),
4751            });
4752            vote_pos.push(j);
4753        }
4754        let allowed = labels.clone();
4755        let mut vote_losses = Vec::new();
4756        let vote_retries = self.state.config.graph.retries;
4757        let vote_cache = self.state.config.cache_dir();
4758        let ctx = WaveCtx {
4759            carry_seats: false,
4760            pauses: Some((&self.pause, &self.interrupt)),
4761            run: &run_id,
4762            node: "vote",
4763            prompts: &prompts,
4764            cache: vote_cache.as_deref(),
4765            round: None,
4766        };
4767        let votes = ask_json_wave::<FinalVote>(
4768            vote_jobs,
4769            Arc::clone(&self.sem),
4770            vote_retries,
4771            &self.roles.judge_roster,
4772            &ctx,
4773            &mut vote_losses,
4774            &mut self.state,
4775            &move |v: &FinalVote| match v.label() {
4776                Some(c) if allowed.contains(&c) => Ok(()),
4777                other => bail!("vote {other:?} is not one of {allowed:?}"),
4778            },
4779        )
4780        .await;
4781        if self.state.resource_block.is_some() {
4782            self.stop_for_resource()?;
4783            return Ok(false);
4784        }
4785        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4786            let agent_id = seat.agent.clone();
4787            self.state.seats.insert(seat.key.clone(), seat);
4788            match res {
4789                Ok((v, _)) => {
4790                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4791                        rec.vote = v.label();
4792                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4793                    } else {
4794                        self.state.votes.push(VoteRecord {
4795                            judge: j + 1,
4796                            agent: agent_id,
4797                            vote: v.label(),
4798                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4799                            changed: false,
4800                        });
4801                    }
4802                    self.state.event(
4803                        "recover",
4804                        format!("judge {} voted again after the limit", j + 1),
4805                    );
4806                }
4807                Err(e) => {
4808                    self.state
4809                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4810                }
4811            }
4812        }
4813
4814        // A seat that ranked again is present even if its re-vote failed —
4815        // `tally` falls back to the initial ranking's first choice — so clear
4816        // its quota loss. Seats that still fail keep theirs and stay absent.
4817        let recovered_keys: BTreeSet<String> = recovered
4818            .iter()
4819            .map(|&j| format!("judge-{}", j + 1))
4820            .collect();
4821        self.state
4822            .quota
4823            .retain(|q| !recovered_keys.contains(&q.seat));
4824        // A seat that hit the limit again is a fresh loss, not the old one:
4825        // replace the stale entry so the history stays one-per-seat and the
4826        // daemon can tell this attempt's loss from a previous session's.
4827        for loss in judge_losses.into_iter().chain(vote_losses) {
4828            if recovered_keys.contains(&loss.seat) {
4829                continue;
4830            }
4831            self.state.quota.retain(|q| q.seat != loss.seat);
4832            self.state.quota.push(loss);
4833        }
4834
4835        // Recompute the verdict from the refreshed panel.
4836        self.state.tally = None;
4837        self.tally()?;
4838        Ok(self
4839            .state
4840            .tally
4841            .as_ref()
4842            .map(|t| t.met_quorum)
4843            .unwrap_or(false))
4844    }
4845
4846    // ---------------------------------------------------------------- graft
4847
4848    /// Port the strengths the judges attributed to the losing candidates onto
4849    /// the winner, once, between the tally and the review loop.
4850    ///
4851    /// Best-effort by construction: it returns nothing and fails nothing. A
4852    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4853    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4854    /// the run carries on as if the phase were off. See [`graft`] for the
4855    /// decisions and `AGENTS.md` for the invariants.
4856    ///
4857    /// The one exception is a rollback that could not be confirmed: the
4858    /// rejected commit may still be the winner's tip, so the run is blocked
4859    /// (resumable; the record stays `Running`, which makes a resume retry the
4860    /// restore) instead of reviewing, gating and merging it.
4861    async fn graft(&mut self) -> Result<()> {
4862        let Err(e) = self.graft_inner().await else {
4863            return Ok(());
4864        };
4865        if e.is::<crate::repo_lock::ResourceBlocked>() {
4866            return Err(e);
4867        }
4868        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4869            self.state.status = RunStatus::Blocked;
4870            self.state.event("graft", format!("blocked: {e:#}"));
4871            self.state.save()?;
4872            return Err(e);
4873        }
4874        // An error after the `Running` record was saved may have left the
4875        // fixer's work in the tree: the record must not stay unsettled
4876        // (a later resume would reset past the review fixes), so confirm the
4877        // restore now, or block.
4878        if let Some(rec) = self.state.graft.clone()
4879            && !rec.outcome.settled()
4880        {
4881            if let Some(winner) = self.state.winner().cloned() {
4882                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4883                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4884                        return Err(re);
4885                    }
4886                    self.state.status = RunStatus::Blocked;
4887                    self.state.event("graft", format!("blocked: {re:#}"));
4888                    self.state.save()?;
4889                    return Err(re);
4890                }
4891            }
4892            if let Some(g) = self.state.graft.as_mut() {
4893                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4894            }
4895        }
4896        self.state.event("graft", format!("skipped: {e:#}"));
4897        if let Err(e) = self.state.save() {
4898            tracing::warn!("save after graft: {e:#}");
4899        }
4900        Ok(())
4901    }
4902
4903    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4904    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4905    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4906        let _coordination = self.coordinate(wt).await?;
4907        crate::recovery::restore_repair(wt, pre)
4908            .await
4909            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4910    }
4911
4912    /// A skip is a run event, recorded once however often the node is re-entered.
4913    fn graft_skipped(&mut self, why: &str) {
4914        let message = format!("skipped: {why}");
4915        if !self
4916            .state
4917            .events
4918            .iter()
4919            .any(|e| e.node == "graft" && e.message == message)
4920        {
4921            self.state.event("graft", message);
4922        }
4923    }
4924
4925    async fn graft_inner(&mut self) -> Result<()> {
4926        // A round that was started and never finished (crash, kill): the
4927        // fixer may have left anything in the tree, so put the winner back.
4928        if let Some(rec) = self.state.graft.clone()
4929            && !rec.outcome.settled()
4930        {
4931            // Reviews exist only if the graft ended without its record being
4932            // settled (a swallowed error): the tree has moved on since, and
4933            // a reset to `pre` would discard the review fixes.
4934            if self.state.reviews.is_empty()
4935                && let Some(winner) = self.state.winner().cloned()
4936            {
4937                self.graft_restore(&winner.worktree, &rec.pre).await?;
4938            }
4939            if rec.outcome == GraftOutcome::WaitingForRestore {
4940                self.state.graft = None;
4941                self.state.save()?;
4942            } else {
4943                if let Some(g) = self.state.graft.as_mut() {
4944                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4945                }
4946                self.state.event(
4947                    "graft",
4948                    "rolled back: the previous round was interrupted; the winner is untouched",
4949                );
4950                self.state.save()?;
4951                return Ok(());
4952            }
4953        }
4954        let plan = match graft::plan(&self.state) {
4955            Ok(plan) => plan,
4956            Err(skip) => {
4957                if skip != graft::Skip::AlreadyDone {
4958                    self.graft_skipped(skip.reason());
4959                    self.state.save()?;
4960                }
4961                return Ok(());
4962            }
4963        };
4964        let Some(winner) = self.state.winner().cloned() else {
4965            return Ok(());
4966        };
4967        let wt = winner.worktree.clone();
4968        if !git::is_clean(&wt).await? {
4969            self.graft_skipped("the winner's worktree has uncommitted changes");
4970            self.state.save()?;
4971            return Ok(());
4972        }
4973        let pre = git::rev_parse(&wt, "HEAD").await?;
4974        // Saved before the fixer is called: a stop from here on is repaired
4975        // by the branch above on the next entry.
4976        self.state.graft = Some(GraftRecord {
4977            pre: pre.clone(),
4978            post: None,
4979            lines: None,
4980            limit: None,
4981            agent: None,
4982            outcome: GraftOutcome::Running,
4983            requested: graft::requested(&plan),
4984            adoption: None,
4985            adoption_error: None,
4986        });
4987        self.state.event(
4988            "graft",
4989            format!(
4990                "porting strengths of candidate(s) {} onto {}",
4991                plan.strengths
4992                    .iter()
4993                    .map(|(l, _, _)| l.to_string())
4994                    .collect::<Vec<_>>()
4995                    .join(", "),
4996                plan.winner
4997            ),
4998        );
4999        self.state.save()?;
5000
5001        let fix_prompt = prompt::graft(
5002            &self.state.instruction,
5003            plan.winner,
5004            &plan.strengths,
5005            &self.state.config.graph.language,
5006        );
5007        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5008        let sessions = self.state.config.graph.sessions;
5009        let artifacts = agent::artifacts_dir(&self.state.dir());
5010        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5011        let (_, seat, out) = self
5012            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5013                SeatJob {
5014                    prompt: fix_prompt.clone(),
5015                    spec,
5016                    seat,
5017                    cwd: wt.clone(),
5018                    timeout,
5019                    allow_write: true,
5020                    sessions,
5021                    artifacts: artifacts.clone(),
5022                    stem: "graft".to_owned(),
5023                    handover: None,
5024                }
5025            })
5026            .await;
5027        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5028            // Preserve the need to restore earlier fallback edits without
5029            // confusing an unstarted seat with an interrupted invocation.
5030            self.state.graft.as_mut().expect("graft record").outcome =
5031                GraftOutcome::WaitingForRestore;
5032            return self.stop_for_resource();
5033        }
5034        let mut why: Option<String> = None;
5035        match &out {
5036            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5037                why = Some("the fixer produced nothing usable".to_owned());
5038            }
5039            AgentOutcome::Ok(_) => {}
5040            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5041            AgentOutcome::Quota(o) => {
5042                self.state.quota.push(QuotaLoss::from_report(
5043                    seat.key.clone(),
5044                    "graft",
5045                    o.quota.as_ref(),
5046                ));
5047                why = Some("rate limited (quota)".to_owned());
5048            }
5049            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5050                why = Some(format!(
5051                    "the fixer failed: {}",
5052                    e.lines()
5053                        .next()
5054                        .unwrap_or("")
5055                        .chars()
5056                        .take(160)
5057                        .collect::<String>()
5058                ));
5059            }
5060        }
5061        let agent_id = seat.agent.clone();
5062        self.state.seats.insert(seat.key.clone(), seat);
5063
5064        let mut post = None;
5065        let mut lines = None;
5066        let _coordination = self.coordinate(&wt).await?;
5067        if why.is_none() {
5068            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5069            if let Ok(r) = &rescued {
5070                self.state.note_withheld("graft", &r.withheld);
5071            }
5072            let head = git::rev_parse(&wt, "HEAD").await?;
5073            // What is measured must be what gets reviewed and gated: a failed
5074            // rescue, or anything still uncommitted (withheld lockfiles
5075            // aside, which are never part of the tree), rejects the graft.
5076            let dirty = match &rescued {
5077                Ok(r) => git::status_porcelain(&wt)
5078                    .await
5079                    .map(|st| {
5080                        st.lines().any(|l| {
5081                            !r.withheld
5082                                .iter()
5083                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5084                        })
5085                    })
5086                    .unwrap_or(true),
5087                Err(_) => true,
5088            };
5089            if dirty {
5090                why = Some("the fixer left changes that could not be committed".to_owned());
5091            } else if head == pre {
5092                why = Some("the fixer changed nothing".to_owned());
5093            } else {
5094                lines = git::changed_lines(&wt, &pre, &head).await;
5095                post = Some(head);
5096            }
5097        }
5098        let (adoption, adoption_error) = match &out {
5099            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5100            _ => (None, None),
5101        };
5102        let requested = graft::requested(&plan);
5103        let outcome = match why {
5104            None => {
5105                self.state.event(
5106                    "graft",
5107                    format!(
5108                        "applied: {}",
5109                        lines.map_or_else(
5110                            || "changed lines unavailable".to_owned(),
5111                            |n| format!("{n} changed lines")
5112                        )
5113                    ),
5114                );
5115                GraftOutcome::Applied
5116            }
5117            Some(why) => {
5118                // Not settled until the restore is confirmed: the record
5119                // stays `Running`, so a resume retries it.
5120                crate::recovery::restore_repair(&wt, &pre)
5121                    .await
5122                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5123                self.state.event(
5124                    "graft",
5125                    format!("rolled back: {why}; the winner is untouched"),
5126                );
5127                GraftOutcome::RolledBack(why)
5128            }
5129        };
5130        self.state.graft = Some(GraftRecord {
5131            pre,
5132            post,
5133            lines,
5134            limit: None,
5135            agent: Some(agent_id),
5136            outcome,
5137            requested,
5138            adoption,
5139            adoption_error,
5140        });
5141        self.state.save()?;
5142        Ok(())
5143    }
5144
5145    // ----------------------------------------------------------------- fold
5146
5147    async fn fold_losers(&mut self) -> Result<()> {
5148        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5149            return Ok(());
5150        };
5151        if !self.state.candidate_preservation.is_empty() {
5152            bail!("candidate preservation incomplete; refusing cleanup");
5153        }
5154        let repo = self.state.repo.clone();
5155        let _coordination = self.coordinate(&repo).await?;
5156        let mut folded = Vec::new();
5157        for i in 0..self.state.candidates.len() {
5158            let c = &self.state.candidates[i];
5159            if c.label == winner || c.folded {
5160                continue;
5161            }
5162            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5163            remove_preserved_worktree(&repo, &wt).await?;
5164            if git::branch_exists(&repo, &branch).await?
5165                && !git::branch_delete(&repo, &branch).await?
5166            {
5167                bail!("candidate branch removal refused");
5168            }
5169            self.state.candidates[i].folded = true;
5170            self.state.save()?;
5171            folded.push(label.to_string());
5172        }
5173        // The judges are finished; their checkouts are pure cost from here.
5174        let root = self.state.worktree_root();
5175        for j in 1..=self.roles.judges.len() {
5176            let wt = root.join(format!("judge-{j}"));
5177            if wt.exists() {
5178                remove_preserved_worktree(&repo, &wt).await?;
5179            }
5180        }
5181        // The design-deliberation stage is finished by the time a tally
5182        // exists — same reasoning as the judges above.
5183        if self.state.config.graph.advise {
5184            for k in 1..=self.state.config.graph.advisors {
5185                let wt = root.join(format!("advisor-{k}"));
5186                if wt.exists() {
5187                    remove_preserved_worktree(&repo, &wt).await?;
5188                }
5189            }
5190        }
5191        if !folded.is_empty() {
5192            self.state
5193                .event("fold", format!("folded candidates {}", folded.join(", ")));
5194            self.state.save()?;
5195        }
5196        Ok(())
5197    }
5198
5199    // ------------------------------------------------------------ base sync
5200
5201    async fn hold_base_sync(&mut self) -> Result<bool> {
5202        self.hold_base_sync_as(None).await
5203    }
5204
5205    async fn hold_base_sync_kind(
5206        &mut self,
5207        kind: crate::base_sync_recovery::StopKind,
5208    ) -> Result<()> {
5209        self.hold_base_sync_as(Some(kind)).await?;
5210        Ok(())
5211    }
5212
5213    async fn hold_base_sync_as(
5214        &mut self,
5215        kind: Option<crate::base_sync_recovery::StopKind>,
5216    ) -> Result<bool> {
5217        if !self.state.base_sync_blocked() {
5218            self.state.base_sync_recovery.stop = None;
5219            return Ok(false);
5220        }
5221        let head = match self.state.winner() {
5222            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5223                .await
5224                .unwrap_or_default(),
5225            None => String::new(),
5226        };
5227        self.state.record_base_sync_stop(&head);
5228        if let Some(kind) = kind {
5229            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
5230        }
5231        self.state.save()?;
5232        Ok(true)
5233    }
5234
5235    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5236    /// anything verifies it.
5237    ///
5238    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5239    /// read whatever is checked out in the winner's worktree. Left alone that
5240    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5241    /// it when the run *branched* - and a run takes long enough that the base
5242    /// has usually moved by the time it gets here. A gate that ran there
5243    /// answers "green on the commit this run started from", not "green on
5244    /// what is about to land", and the difference showed up three times in
5245    /// one day as a green run whose merge would have reverted a file another
5246    /// pull request had already landed.
5247    ///
5248    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5249    /// `land::Step::Rebase` calls, rather than a second implementation of the
5250    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5251    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5252    /// doc warns about ("two answers to a question nobody notices until a
5253    /// diff is wrong").
5254    ///
5255    /// Foreign-checkout reviews instead merge the base into the owned shadow
5256    /// worktree, preserving the ancestry required for fast-forward delivery.
5257    /// A conflicting merge is aborted and held for operator resolution.
5258    ///
5259    /// A conflict is not the end of the road: the standing rebase is handed
5260    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5261    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5262    /// land). Once it finishes, review and the gate run as usual on the
5263    /// rebased tree, which is where a breakage the new base caused is caught
5264    /// by the ordinary gate-fix round. magi resolves nothing itself.
5265    ///
5266    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5267    /// `state.base_sync.attempts`, is how many times the base is *rebased
5268    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
5269    /// *conflict* was given to a fixer. When the fixer cannot finish the
5270    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5271    /// what was tried (rounds spent, paths still conflicted) and the branch
5272    /// and worktree stay exactly as they were - untouched, for a person to
5273    /// look at - which is also what makes re-entering this function
5274    /// afterwards a no-op instead of a second attempt at the same wall. A
5275    /// push failure ends the same way.
5276    async fn sync_to_base(&mut self) -> Result<()> {
5277        if self.state.status == RunStatus::AlreadyInBase {
5278            return Ok(());
5279        }
5280        self.hold_base_sync().await?;
5281        let conflicted = self
5282            .state
5283            .base_sync
5284            .as_ref()
5285            .is_some_and(|s| s.conflict.is_some());
5286        let Some(winner) = self.state.winner().cloned() else {
5287            return Ok(());
5288        };
5289
5290        let repo = self.state.repo.clone();
5291        let remote = self.state.config.merge.remote.clone();
5292        let base_branch = self.state.base_branch.clone();
5293        let tracking = format!("{remote}/{base_branch}");
5294
5295        let fetched_base =
5296            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5297        // No network, or the remote never had this branch: the run already
5298        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5299        // otherwise), and one that got this far is not blocked by a fetch
5300        // that fails now. It just has no newer tip to compare against.
5301        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
5302            return Ok(());
5303        };
5304
5305        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5306        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5307        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5308
5309        // Before any rebase, and before a recorded conflict is honoured: a
5310        // branch whose change reached the base under other commit ids has
5311        // nothing to rebase and nothing to conflict with, and a run that
5312        // already stopped on that phantom conflict recovers here on resume.
5313        // `behind == 0` with head == tip is a branch the base has since taken
5314        // in whole, whether or not a conflict was ever recorded: the ancestry
5315        // proof must still run (`classify` ignores a head still on the start
5316        // commit).
5317        if (behind > 0 || conflicted || head == tip)
5318            && self
5319                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5320                .await?
5321        {
5322            return Ok(());
5323        }
5324        if conflicted && self.state.base_recovery_pending() {
5325            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5326            let owner =
5327                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5328            if !fetched_base
5329                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5330                || !git::is_clean(&winner.worktree).await?
5331                || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
5332            {
5333                return Ok(());
5334            }
5335        }
5336        let recovery = conflicted && self.state.consume_base_recovery(&head);
5337        if conflicted {
5338            // Consumption (including a refused changed HEAD) is durable before Git.
5339            self.state.save()?;
5340            if !recovery {
5341                return Ok(());
5342            }
5343        }
5344
5345        if behind == 0 {
5346            // A fixer-finished rebase moves the branch ref before the
5347            // winner's worktree is told (`sync_to_head` below). A run that
5348            // died in between resumes here with `behind == 0` and a tree still
5349            // holding the pre-rebase files, which review and the gate would
5350            // then read. That state is exactly: HEAD moved off the tip the
5351            // rebase started from, yet the tree is still identical to that
5352            // tip. A tree with edits of its own differs from it, so nothing
5353            // is thrown away.
5354            if let Some(from) = self
5355                .state
5356                .rebase_fixes
5357                .iter()
5358                .rev()
5359                .find_map(|r| r.from.clone())
5360                && from != head
5361                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5362                    .await
5363                    .is_ok_and(|o| o.ok())
5364            {
5365                git::sync_to_head(&winner.worktree).await?;
5366            }
5367            // An earlier attempt may have rebased the branch locally and died
5368            // before pushing it (only the fresh-rebase arm below pushes).
5369            // Publish it now, so the plain push at PR time is not refused as
5370            // a non-fast-forward. A run already holding a recorded conflict
5371            // never reaches here; that case is out of scope.
5372            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5373            if let Some(why) = &conflict {
5374                self.state.status = RunStatus::Blocked;
5375                self.state.event("land", why.clone());
5376            }
5377            self.state.base_sync = Some(BaseSync {
5378                tip,
5379                behind: 0,
5380                attempts: attempts + usize::from(recovery),
5381                conflict,
5382                already_in: None,
5383            });
5384            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5385                .await?;
5386            self.state.save()?;
5387            return Ok(());
5388        }
5389
5390        if attempts >= BASE_SYNC_ROUNDS && !recovery {
5391            let why = format!(
5392                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5393                 synchronization(s); synchronizing again would only race it",
5394                winner.branch
5395            );
5396            self.state.status = RunStatus::Blocked;
5397            self.state.base_sync = Some(BaseSync {
5398                tip,
5399                behind,
5400                attempts,
5401                conflict: Some(why.clone()),
5402                already_in: None,
5403            });
5404            self.state.event("land", why);
5405            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5406                .await?;
5407            self.state.save()?;
5408            return Ok(());
5409        }
5410
5411        if self.state.foreign_checkout.is_some() {
5412            // Delivery to the operator's checkout is fast-forward only. A
5413            // rebase here would discard its saved head from our ancestry.
5414            let wait = repository_wait(&self.state, None);
5415            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5416            let merged = git::git_raw(
5417                &winner.worktree,
5418                &[
5419                    "-c",
5420                    "core.hooksPath=/dev/null",
5421                    "merge",
5422                    "--no-ff",
5423                    "--no-autostash",
5424                    "--no-edit",
5425                    "-m",
5426                    &format!("Merge {tracking} before verifying"),
5427                    &tip,
5428                ],
5429            )
5430            .await?;
5431            let conflict = if merged.ok() {
5432                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5433                self.state.event(
5434                    "land",
5435                    format!("merged {tracking} into {} before verifying", winner.branch),
5436                );
5437                None
5438            } else {
5439                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5440                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5441                }
5442                let why = format!(
5443                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5444                    winner.branch,
5445                    merged.stderr.chars().take(600).collect::<String>()
5446                );
5447                self.state.status = RunStatus::Blocked;
5448                self.state.event("land", &why);
5449                Some(why)
5450            };
5451            let synced = conflict.is_none();
5452            self.state.base_sync = Some(BaseSync {
5453                tip,
5454                behind: if synced { 0 } else { behind },
5455                attempts: attempts + 1,
5456                conflict,
5457                already_in: None,
5458            });
5459            self.state.save()?;
5460            return Ok(());
5461        }
5462
5463        self.state.event(
5464            "land",
5465            format!(
5466                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5467                winner.branch
5468            ),
5469        );
5470        self.state.save()?;
5471
5472        // The remote's copy of the branch, read now and only if the fetch
5473        // really succeeded (a stale tracking ref must never pin a lease). It is
5474        // pushed over after a rebase only when it is a commit this branch
5475        // already contains, by ancestry or by patch (an earlier rebase of ours
5476        // that never reached the remote): anything else is somebody else's work.
5477        let branch_tracking = format!("{remote}/{}", winner.branch);
5478        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5479        let remote_tip = if self.state.task_pr_branch().is_none()
5480            && matches!(&fetched_branch, Ok(o) if o.ok())
5481        {
5482            git::rev_parse(&repo, &branch_tracking).await.ok()
5483        } else {
5484            None
5485        };
5486        // A remote tip this branch does not contain is somebody else's work:
5487        // rebasing would leave a local tip that can never be pushed. Stop
5488        // before touching anything and say so.
5489        if let Some(theirs) = &remote_tip
5490            && !git::is_ancestor(&repo, theirs, &head).await
5491            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5492                .await
5493                .is_ok_and(|missing| missing.is_empty())
5494        {
5495            let why = format!(
5496                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5497                 them",
5498                short(theirs),
5499                winner.branch
5500            );
5501            self.state.status = RunStatus::Blocked;
5502            self.state.base_sync = Some(BaseSync {
5503                tip,
5504                behind,
5505                attempts: attempts + usize::from(recovery),
5506                conflict: Some(why.clone()),
5507                already_in: None,
5508            });
5509            self.state.event("land", why);
5510            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5511                .await?;
5512            self.state.save()?;
5513            return Ok(());
5514        }
5515
5516        let scratch = self.state.dir().join("base-sync");
5517        let rebased =
5518            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5519                .await
5520            {
5521                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5522                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5523                Err(e) => Err(e),
5524            };
5525        let attempts = attempts + 1;
5526        let stop_kind = match &rebased {
5527            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5528            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5529            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5530        };
5531        match rebased {
5532            Ok(None) => {
5533                // The branch ref moved, but a worktree that already had it
5534                // checked out (the winner's) was not told; sync its index and
5535                // files before anything reads them.
5536                git::sync_to_head(&winner.worktree).await?;
5537                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5538                let mut conflict = None;
5539                if let Some(pinned) = &remote_tip {
5540                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5541                    match pushed {
5542                        Ok(o) if o.ok() => self.state.event(
5543                            "land",
5544                            format!("pushed rebased {} to {remote}", winner.branch),
5545                        ),
5546                        Ok(o) => {
5547                            conflict = Some(format!(
5548                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5549                                winner.branch,
5550                                short(pinned),
5551                                o.stderr.chars().take(600).collect::<String>()
5552                            ));
5553                        }
5554                        Err(e) => {
5555                            conflict = Some(format!(
5556                                "rebased {} locally but could not push it: {e:#}",
5557                                winner.branch
5558                            ));
5559                        }
5560                    }
5561                }
5562                if let Some(why) = &conflict {
5563                    self.state.status = RunStatus::Blocked;
5564                    self.state.event("land", why.clone());
5565                }
5566                self.state.base_sync = Some(BaseSync {
5567                    tip: tip.clone(),
5568                    behind: 0,
5569                    attempts,
5570                    conflict,
5571                    already_in: None,
5572                });
5573                self.state
5574                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
5575            }
5576            Ok(Some(conflict)) => {
5577                let why = format!(
5578                    "{} conflicts with {tracking} and did not rebase: {}",
5579                    winner.branch,
5580                    conflict.chars().take(600).collect::<String>()
5581                );
5582                self.state.status = RunStatus::Blocked;
5583                self.state.base_sync = Some(BaseSync {
5584                    tip,
5585                    behind,
5586                    attempts,
5587                    conflict: Some(why.clone()),
5588                    already_in: None,
5589                });
5590                self.state.event("land", why);
5591            }
5592            // A refused Git write or a busy repository: nothing is wrong with
5593            // the branch. The scratch rebase stays standing and the run
5594            // resumes into it (`base_sync` must not claim a conflict).
5595            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5596                self.state.save()?;
5597                return Ok(());
5598            }
5599            Err(e) => {
5600                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5601                self.state.status = RunStatus::Blocked;
5602                self.state.base_sync = Some(BaseSync {
5603                    tip,
5604                    behind,
5605                    attempts,
5606                    conflict: Some(why.clone()),
5607                    already_in: None,
5608                });
5609                self.state.event("land", why);
5610            }
5611        }
5612        if !self.state.base_sync_blocked() {
5613            self.state.base_sync_recovery.stop = None;
5614        }
5615        self.hold_base_sync_kind(stop_kind).await?;
5616        self.state.save()?;
5617        Ok(())
5618    }
5619
5620    /// Push a branch an earlier attempt rebased locally but never published,
5621    /// pinned to the remote tip read right after a successful fetch. Returns
5622    /// the reason when the run must stop; `None` when there was nothing to do
5623    /// (no remote copy, the same tip, or a remote copy this branch already
5624    /// contains, which the PR-time push fast-forwards) or the push succeeded.
5625    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5626        // Task competitions publish only after the PR text gate, to their
5627        // public head; never resurrect a candidate branch on the remote.
5628        if self.state.task_pr_branch().is_some() {
5629            return None;
5630        }
5631        let repo = self.state.repo.clone();
5632        let remote = self.state.config.merge.remote.clone();
5633        let fetched = git::fetch(&repo, &remote, branch).await;
5634        if !matches!(&fetched, Ok(o) if o.ok()) {
5635            return None;
5636        }
5637        let branch_tracking = format!("{remote}/{branch}");
5638        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5639        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5640            return None;
5641        }
5642        if !crate::reconcile::origin_missing(&repo, head, &theirs)
5643            .await
5644            .is_ok_and(|missing| missing.is_empty())
5645        {
5646            return Some(format!(
5647                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5648                 them",
5649                short(&theirs)
5650            ));
5651        }
5652        match git::push_pinned(&repo, &remote, branch, &theirs).await {
5653            Ok(o) if o.ok() => {
5654                self.state
5655                    .event("land", format!("pushed rebased {branch} to {remote}"));
5656                None
5657            }
5658            Ok(o) => Some(format!(
5659                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5660                 someone may have pushed): {}",
5661                short(&theirs),
5662                o.stderr.chars().take(600).collect::<String>()
5663            )),
5664            Err(e) => Some(format!(
5665                "{branch} is rebased locally but could not be pushed: {e:#}"
5666            )),
5667        }
5668    }
5669
5670    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
5671    /// is already on `tip` under other commit ids ([`crate::already`]); returns
5672    /// whether it did.
5673    ///
5674    /// Checked only when the base is ahead of the branch. A failing check is
5675    /// "not proven" - the ordinary rebase path then decides - never a reason to
5676    /// stop the run.
5677    ///
5678    /// The remote copy of the branch is held to the same standard as the local
5679    /// one: if it carries a tip this worktree does not, that tip must itself be
5680    /// proven in the base, or nothing is settled (a pull request would
5681    /// otherwise be closed over commits nobody checked). The pull request is
5682    /// closed *before* the terminal status is saved; if that fails for a
5683    /// reason other than a refusal (no network, a `gh` error) the run is left
5684    /// `Blocked` with the reason as its conflict, which a resume retries -
5685    /// the same recovery a phantom conflict gets.
5686    async fn settle_already_in(
5687        &mut self,
5688        branch: &str,
5689        tip: &str,
5690        head: &str,
5691        attempts: usize,
5692        behind: usize,
5693    ) -> Result<bool> {
5694        let repo = self.state.repo.clone();
5695        let remote = self.state.config.merge.remote.clone();
5696        let start = self.state.base_commit.clone();
5697        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5698            Ok(Some(e)) => e,
5699            Ok(None) => return Ok(false),
5700            Err(e) => {
5701                tracing::warn!("already-in-base check for {branch}: {e:#}");
5702                return Ok(false);
5703            }
5704        };
5705        let mut verified = vec![head.to_owned()];
5706        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5707        let fetched = git::fetch(&repo, &remote, &public_branch).await;
5708        if matches!(&fetched, Ok(o) if o.ok())
5709            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5710            && theirs != head
5711        {
5712            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5713                Ok(Some(_)) => verified.push(theirs),
5714                _ => return Ok(false),
5715            }
5716        }
5717        let base_branch = self.state.base_branch.clone();
5718        let message = format!(
5719            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5720             land",
5721            evidence.names(),
5722            evidence.proof.as_str()
5723        );
5724        let closed =
5725            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5726                .await;
5727        match closed {
5728            Ok(Ok(url)) => self
5729                .state
5730                .event("land", format!("closed {url}: superseded on {base_branch}")),
5731            Ok(Err(why)) => self
5732                .state
5733                .event("land", format!("did not close a pull request: {why}")),
5734            Err(e) => {
5735                let why = format!(
5736                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
5737                     not be closed ({e:#}); resume to retry"
5738                );
5739                self.state.status = RunStatus::Blocked;
5740                self.state.base_sync = Some(BaseSync {
5741                    tip: tip.to_owned(),
5742                    behind,
5743                    attempts,
5744                    conflict: Some(why.clone()),
5745                    already_in: None,
5746                });
5747                self.state.event("land", why);
5748                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5749                    .await?;
5750                self.state.save()?;
5751                return Ok(true);
5752            }
5753        }
5754        self.state.status = RunStatus::AlreadyInBase;
5755        self.state.base_sync_recovery.stop = None;
5756        self.state.base_sync = Some(BaseSync {
5757            tip: tip.to_owned(),
5758            behind,
5759            attempts,
5760            conflict: None,
5761            already_in: Some(evidence),
5762        });
5763        self.state.event("land", message);
5764        self.state.save()?;
5765        self.settle_questions();
5766        Ok(true)
5767    }
5768
5769    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
5770    /// last landed the winner on, once it has run, else the commit the run
5771    /// branched from.
5772    ///
5773    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
5774    /// and `vote` all happen before there is a winner to rebase, so they
5775    /// compare every candidate against the branch point on purpose, and a
5776    /// base that moves after they are already done cannot change an answer
5777    /// they already gave.
5778    fn landing_base(&self) -> String {
5779        self.state
5780            .base_sync
5781            .as_ref()
5782            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5783    }
5784
5785    // ------------------------------------------------------- operator fix
5786
5787    /// Route specific, already-recorded review findings to a fixer for a
5788    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5789    /// entry point.
5790    ///
5791    /// Distinct from `review_loop`'s own fix step in three ways: it never
5792    /// runs a reviewer wave, it never spends review-round budget, and what
5793    /// happened is recorded as an [`OperatorFixRequest`] appended to
5794    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5795    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5796    /// and vote must never be rewritten to look like a manufactured blocking
5797    /// verdict.
5798    ///
5799    /// Only meaningful once review has actually concluded: `Ready` (handed
5800    /// off with findings still open, or simply concluded clean while minor
5801    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5802    /// gate failed). Everything else is refused: a run still in progress
5803    /// should simply be resumed, and a `Merged` run's branch has already
5804    /// landed — reopening *this* run's own record cannot change that, so the
5805    /// answer there is a fresh `magi review <branch>`.
5806    ///
5807    /// A real commit here re-verifies through a fresh, ordinary review-only
5808    /// run on the same branch ([`Self::review`]) rather than reopening this
5809    /// run's own `review_loop`: once any round in this run's history went
5810    /// clean, `review_conclusion` treats that as permanent by design (the
5811    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5812    /// way to force one more genuine reviewer wave out of *this* run without
5813    /// either rewriting history or weakening that guarantee for every other
5814    /// caller. A review-only run costs nothing extra — no implementation, no
5815    /// judging, no vote — and exercises the exact same review → verify →
5816    /// gate → (human) merge path, unmodified.
5817    pub async fn fix_selected(
5818        &mut self,
5819        ids: &[String],
5820        reason: &str,
5821        allow_stale: bool,
5822    ) -> Result<()> {
5823        let reason = reason.trim();
5824        if reason.is_empty() {
5825            bail!("a fix request needs a reason — that is the operator's own record of why");
5826        }
5827        if ids.is_empty() {
5828            bail!("no finding id given");
5829        }
5830        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5831            bail!(
5832                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5833                 has already concluded — can be given a targeted fix. A run still \
5834                 in progress should simply be resumed; a `merged` run's branch has \
5835                 already landed, so its answer is a fresh `magi review <branch>`, \
5836                 not reopening this run's own record",
5837                self.state.id,
5838                self.state.status.as_str()
5839            );
5840        }
5841        let Some(winner) = self.state.winner().cloned() else {
5842            bail!("run {} has no winning candidate to fix", self.state.id);
5843        };
5844        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5845            bail!(
5846                "branch `{}` no longer exists; this run cannot be extended",
5847                winner.branch
5848            );
5849        }
5850        let home = crate::run::home();
5851        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5852            bail!(
5853                "run {} is currently being worked on by another magi process",
5854                self.state.id
5855            );
5856        }
5857        // Held for the rest of this call, including the follow-up review
5858        // below: two `magi fix` invocations against the same run must not
5859        // both reach the worktree manipulation further down, which would
5860        // otherwise race to remove and recreate the same directory — see
5861        // [`FixClaim`]'s own doc.
5862        let _claim = FixClaim::acquire(&self.state.dir())?;
5863
5864        // Resolve every id before spending anything — an unknown id refuses
5865        // the whole request rather than silently dropping it — and dedup
5866        // while keeping the operator's own order.
5867        let mut seen = BTreeSet::new();
5868        let mut findings = Vec::new();
5869        let mut missing = Vec::new();
5870        for id in ids {
5871            if !seen.insert(id.clone()) {
5872                continue;
5873            }
5874            match self.state.finding(id) {
5875                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5876                    id: f.id.clone(),
5877                    severity: f.severity,
5878                    reviewer_vote: rec.vote,
5879                    round: round.round,
5880                    round_head: round.head.clone(),
5881                    reviewer: rec.reviewer,
5882                    agent: rec.agent.clone(),
5883                    file: f.file.clone(),
5884                    line: f.line,
5885                    title: f.title.clone(),
5886                    detail: f.detail.clone(),
5887                    outcome: OperatorFixOutcome::Pending,
5888                }),
5889                None => missing.push(id.clone()),
5890            }
5891        }
5892        if !missing.is_empty() {
5893            bail!(
5894                "unknown finding id(s): {}; nothing was changed",
5895                missing.join(", ")
5896            );
5897        }
5898
5899        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5900        let stale_details: Vec<(String, String)> = findings
5901            .iter()
5902            .filter(|f| f.round_head != head_at_request)
5903            .map(|f| (f.id.clone(), f.round_head.clone()))
5904            .collect();
5905        let stale = !stale_details.is_empty();
5906        if stale && !allow_stale {
5907            bail!(
5908                "the branch has moved since some finding(s) were raised — {} — now \
5909                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5910                stale_details
5911                    .iter()
5912                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5913                    .collect::<Vec<_>>()
5914                    .join(", "),
5915                short(&head_at_request)
5916            );
5917        }
5918
5919        let request = OperatorFixRequest {
5920            requested_at: Timestamp::now(),
5921            reason: reason.to_owned(),
5922            findings,
5923            head_at_request: head_at_request.clone(),
5924            allow_stale,
5925            stale,
5926            fix: None,
5927            result_head: None,
5928            follow_up_review_run: None,
5929        };
5930        self.state.event(
5931            "fix",
5932            format!(
5933                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5934                request.findings.len(),
5935                request
5936                    .findings
5937                    .iter()
5938                    .map(|f| f.id.as_str())
5939                    .collect::<Vec<_>>()
5940                    .join(", "),
5941            ),
5942        );
5943        // Recorded now, before any worktree work or the fixer call itself —
5944        // and re-saved at each checkpoint below: a crash at any point after
5945        // this (mid fixer call, mid follow-up review) must not lose the fact
5946        // that this was requested, for which findings, and why. Everything
5947        // past this point reads and writes through `request_index` rather
5948        // than a local variable, since `request` itself is moved here.
5949        self.state.operator_fixes.push(request);
5950        self.state.save()?;
5951        let request_index = self.state.operator_fixes.len() - 1;
5952
5953        // A fresh, dedicated worktree for this one call, never the winner's
5954        // own worktree in place: that one may already be gone (folded away),
5955        // and reusing it in place would leave the branch checked out there
5956        // when the follow-up review below tries to check it out again. Freed
5957        // immediately after, either way — but only once confirmed clean:
5958        // `worktree_remove` is a `git worktree remove --force`, which would
5959        // otherwise discard uncommitted work left there by the operator or
5960        // another process before this had a chance to even look at it.
5961        let repo = self.state.repo.clone();
5962        let coordination = self.coordinate(&repo).await?;
5963        if winner.worktree.exists() {
5964            // Lockfiles a rescue commit withheld stay untracked on purpose and
5965            // are already recorded; they are not the operator's work to protect.
5966            let dirty = git::git(
5967                &winner.worktree,
5968                &["status", "--porcelain", "--untracked-files=all"],
5969            )
5970            .await?;
5971            let only_withheld = dirty.lines().all(|l| {
5972                l.strip_prefix("?? ")
5973                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5974            });
5975            if !only_withheld {
5976                bail!(
5977                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5978                     discard them first",
5979                    winner.worktree.display()
5980                );
5981            }
5982            git::worktree_remove(&self.state.repo, &winner.worktree)
5983                .await
5984                .ok();
5985        }
5986        let fix_worktree = self.state.worktree_root().join("operator-fix");
5987        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5988        git::git(
5989            &self.state.repo,
5990            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5991        )
5992        .await
5993        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5994        if !git::is_clean(&fix_worktree).await? {
5995            git::worktree_remove(&self.state.repo, &fix_worktree)
5996                .await
5997                .ok();
5998            bail!(
5999                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6000                winner.branch
6001            );
6002        }
6003        let repo_for_setup = self.state.repo.clone();
6004        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6005            Ok(()) => {
6006                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6007            }
6008            Err(e) => Err(e),
6009        };
6010        if let Err(e) = setup {
6011            git::worktree_remove(&self.state.repo, &fix_worktree)
6012                .await
6013                .ok();
6014            return Err(e);
6015        }
6016
6017        // This explicit operator request retries availability independently
6018        // of execute(); the previous call has already saved and cleaned up.
6019        self.state.resource_block = None;
6020
6021        let run_id = self.state.id.clone();
6022        let prompts = self.state.config.prompts.clone();
6023        let language = self.state.config.graph.language.clone();
6024        let sessions = self.state.config.graph.sessions;
6025        let artifacts = agent::artifacts_dir(&self.state.dir());
6026        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6027            .findings
6028            .iter()
6029            .map(|f| Finding {
6030                id: f.id.clone(),
6031                severity: f.severity,
6032                file: f.file.clone(),
6033                line: f.line,
6034                title: f.title.clone(),
6035                detail: f.detail.clone(),
6036            })
6037            .collect();
6038        let fix_prompt = prompt::operator_fix_in(
6039            &self.state.instruction,
6040            &finding_list,
6041            reason,
6042            &stale_details,
6043            &head_at_request,
6044            &language,
6045            &self.state.config.graph.github_language,
6046        );
6047        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6048        drop(coordination);
6049        let (job, seat, out) = self
6050            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6051                prompt: fix_prompt.clone(),
6052                spec,
6053                seat,
6054                cwd: fix_worktree.clone(),
6055                timeout,
6056                allow_write: true,
6057                sessions,
6058                artifacts: artifacts.clone(),
6059                stem: "operator-fix".to_owned(),
6060                handover: None,
6061            })
6062            .await;
6063        let agent_id = seat.agent.clone();
6064
6065        let mut fix = FixRecord {
6066            agent: agent_id,
6067            addressed: Vec::new(),
6068            rejected: Vec::new(),
6069            notes: String::new(),
6070            committed: false,
6071            failed: None,
6072            duration_ms: 0,
6073            continuation: None,
6074        };
6075        let mut final_seat = seat.clone();
6076        match out {
6077            AgentOutcome::Ok(o) => {
6078                fix.duration_ms = o.duration_ms;
6079                let parsed = verdict::extract_json::<FixReport>(&o.text);
6080                let incomplete_reason = match &parsed {
6081                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6082                        "the reply parsed, but it reported a command whose own CLI \
6083                         never confirmed an exit status"
6084                            .to_owned(),
6085                    ),
6086                    Ok(_) => None,
6087                    Err(e) => Some(e.to_string()),
6088                };
6089                match incomplete_reason {
6090                    None => {
6091                        let report = parsed.expect("checked Ok above");
6092                        fix.addressed = report.addressed;
6093                        fix.rejected = report.rejected;
6094                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6095                    }
6096                    Some(reason) => {
6097                        let (resumed_seat, resolved, failure, cont) = self
6098                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6099                            .await;
6100                        fix.duration_ms += cont.cumulative_wait_ms;
6101                        fix.continuation = Some(cont);
6102                        final_seat = resumed_seat;
6103                        match resolved {
6104                            Some(report) => {
6105                                fix.addressed = report.addressed;
6106                                fix.rejected = report.rejected;
6107                                fix.notes =
6108                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6109                            }
6110                            None => fix.failed = failure,
6111                        }
6112                    }
6113                }
6114            }
6115            AgentOutcome::Dropped(o) => {
6116                fix.duration_ms = o.duration_ms;
6117                let why = o
6118                    .dropped
6119                    .as_ref()
6120                    .map(|d| d.why.as_str())
6121                    .unwrap_or("the CLI ended the stream without delivering its answer");
6122                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6123            }
6124            AgentOutcome::Quota(o) => {
6125                self.state.quota.push(QuotaLoss::from_report(
6126                    final_seat.key.clone(),
6127                    "fix",
6128                    o.quota.as_ref(),
6129                ));
6130                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6131            }
6132            AgentOutcome::Failed(e) => fix.failed = Some(e),
6133            AgentOutcome::ResourceBlocked(e) => {
6134                fix.failed = Some(format!("the fixer was not started: {e}"));
6135            }
6136        }
6137        // Even a deferred call must rescue edits, release the worktree and
6138        // finalize its record below so an explicit `magi fix` can retry.
6139        if fix.continuation.is_none() {
6140            fix.continuation = Some(ContinuationRecord::not_needed());
6141        }
6142        self.state.seats.insert(final_seat.key.clone(), final_seat);
6143
6144        let rescue_message = format!(
6145            "magi: operator-selected fix ({}) (uncommitted work)",
6146            self.state.operator_fixes[request_index]
6147                .findings
6148                .iter()
6149                .map(|f| f.id.as_str())
6150                .collect::<Vec<_>>()
6151                .join(", ")
6152        );
6153        let repo = self.state.repo.clone();
6154        let coordination = self.coordinate(&repo).await?;
6155        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6156            self.state.note_withheld("fix", &r.withheld);
6157        }
6158        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6159        fix.committed = after != head_at_request;
6160        git::worktree_remove(&self.state.repo, &fix_worktree)
6161            .await
6162            .ok();
6163        drop(coordination);
6164
6165        self.state.event(
6166            "fix",
6167            match &fix.failed {
6168                Some(reason) => format!(
6169                    "operator fix: adoption report was lost ({reason}); {}",
6170                    if fix.committed {
6171                        "committed"
6172                    } else {
6173                        "NO new commit"
6174                    }
6175                ),
6176                None => format!(
6177                    "operator fix: {} addressed, {} rejected, {}",
6178                    fix.addressed.len(),
6179                    fix.rejected.len(),
6180                    if fix.committed {
6181                        "committed"
6182                    } else {
6183                        "NO new commit"
6184                    }
6185                ),
6186            },
6187        );
6188
6189        // Every selected finding gets an outcome — never left `Pending` once
6190        // the fixer's own turn is over. A report that never came back at all
6191        // marks every one of them `Unreported`, not silently "not addressed":
6192        // quota, a dropped stream, or an exhausted continuation are gaps in
6193        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6194        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6195        for f in &mut self.state.operator_fixes[request_index].findings {
6196            f.outcome = if fix.failed.is_some() {
6197                OperatorFixOutcome::Unreported
6198            } else if fix.addressed.contains(&f.id) {
6199                OperatorFixOutcome::Addressed
6200            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6201                OperatorFixOutcome::Rejected { why: r.why.clone() }
6202            } else {
6203                OperatorFixOutcome::Unreported
6204            };
6205        }
6206
6207        let committed = fix.committed;
6208        if committed {
6209            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6210        }
6211        self.state.operator_fixes[request_index].fix = Some(fix);
6212        // Saved again now that the fixer's own outcome is final, on top of
6213        // the save right after the request was first pushed above.
6214        self.state.save()?;
6215        if self.state.resource_block.is_some() {
6216            return self.stop_for_resource();
6217        }
6218
6219        if committed {
6220            self.state.event(
6221                "fix",
6222                format!(
6223                    "operator fix committed {}; opening a follow-up review-only run",
6224                    short(&after)
6225                ),
6226            );
6227            // The operator asked for the fix, and the follow-up serves whatever
6228            // task the run it follows served.
6229            let origin =
6230                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6231            match Self::review(
6232                &self.state.repo,
6233                &winner.branch,
6234                self.state.config.clone(),
6235                origin,
6236            )
6237            .await
6238            {
6239                Ok(mut follow_up) => {
6240                    follow_up.state.event(
6241                        "start",
6242                        format!(
6243                            "requested by an operator fix on run {} for finding(s) {}",
6244                            self.state.id,
6245                            self.state.operator_fixes[request_index]
6246                                .findings
6247                                .iter()
6248                                .map(|f| f.id.as_str())
6249                                .collect::<Vec<_>>()
6250                                .join(", "),
6251                        ),
6252                    );
6253                    follow_up.state.save()?;
6254                    let follow_up_id = follow_up.state.id.clone();
6255                    // The follow-up is a run like any other: it belongs to the
6256                    // task of the run it follows, or to one filed for it.
6257                    let adopted = match crate::direct::adopt(
6258                        &follow_up.state,
6259                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6260                    ) {
6261                        Ok(a) => a,
6262                        Err(e) => {
6263                            // An ownerless run must not spend agent calls; it
6264                            // stays saved, and `magi run --resume` adopts it.
6265                            self.state.event(
6266                                "fix",
6267                                format!(
6268                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6269                                ),
6270                            );
6271                            self.state.save()?;
6272                            return Ok(());
6273                        }
6274                    };
6275                    let executed = follow_up.execute().await;
6276                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6277                    if let Some(a) = adopted {
6278                        a.finish(&follow_up.state, executed);
6279                    }
6280                    if let Some(e) = failure {
6281                        self.state.event(
6282                            "fix",
6283                            format!(
6284                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6285                            ),
6286                        );
6287                    }
6288                    self.state.operator_fixes[request_index].follow_up_review_run =
6289                        Some(follow_up_id);
6290                }
6291                Err(e) => {
6292                    self.state.event(
6293                        "fix",
6294                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6295                    );
6296                }
6297            }
6298            self.state.save()?;
6299        }
6300
6301        Ok(())
6302    }
6303
6304    // --------------------------------------------------------------- review
6305
6306    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6307    /// names, in order, each at most once, moving on only when the call
6308    /// advances (an error, a quota hit or nothing usable - the same decision
6309    /// point `agent::chain_advances` is for the other chained roles).
6310    ///
6311    /// Each agent gets its own seat from `Runner::seat`: the same agent
6312    /// continues its conversation (the winner's own implementer seat when it
6313    /// is the winner's author, now that the competition is over), another
6314    /// takes a fresh one so the full prompt is sent again. A handover is
6315    /// recorded under `node`, which is what makes the fallback stick for the
6316    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6317    /// is returned, so an exhausted chain reads exactly like a single failed
6318    /// fixer: one quota loss, the same wording, the same refund. An earlier
6319    /// attempt's edits are left in the tree and judged, with the final
6320    /// attempt's, by what git says afterwards.
6321    ///
6322    /// Returns the job that produced the outcome, for `continue_fix_report`.
6323    async fn ask_fixer(
6324        &mut self,
6325        winner: &Candidate,
6326        node: &'static str,
6327        round: Option<usize>,
6328        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6329    ) -> (SeatJob, SeatState, AgentOutcome) {
6330        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6331        self.ask_fixer_among(attempts, node, round, None, build)
6332            .await
6333    }
6334
6335    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6336    /// worktree and a commit: before the next agent is asked after one that
6337    /// did not satisfy the call, the worktree is put back on that commit, so
6338    /// a failed agent's partial edits are not carried into the next one.
6339    async fn ask_fixer_among(
6340        &mut self,
6341        attempts: Vec<(AgentSpec, String)>,
6342        node: &'static str,
6343        round: Option<usize>,
6344        restore: Option<(&Path, &str)>,
6345        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6346    ) -> (SeatJob, SeatState, AgentOutcome) {
6347        let run_id = self.state.id.clone();
6348        let prompts = self.state.config.prompts.clone();
6349        let cache = self.state.config.cache_dir();
6350        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6351        let mut last = None;
6352        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6353            let seat = self.seat(&key, &spec.id);
6354            let mut job = build(spec.clone(), seat);
6355            if i > 0 {
6356                job.stem = format!("{}-{}", job.stem, spec.id);
6357            }
6358            let ctx = WaveCtx {
6359                carry_seats: false,
6360                pauses: Some((&self.pause, &self.interrupt)),
6361                run: &run_id,
6362                node,
6363                prompts: &prompts,
6364                cache: cache.as_deref(),
6365                round,
6366            };
6367            let (seat, out) =
6368                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6369            // A resource block never advances the chain: the chosen agent
6370            // was not asked, so it must be the one asked again on resume.
6371            let advances = match &out {
6372                AgentOutcome::Ok(o) => agent::output_advances(o),
6373                AgentOutcome::ResourceBlocked(_) => false,
6374                _ => true,
6375            };
6376            if let Some(next) = ids.get(i + 1)
6377                && advances
6378            {
6379                self.state.seats.insert(seat.key.clone(), seat.clone());
6380                let class =
6381                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6382                let reason = match &out {
6383                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6384                    other => fail_reason(other),
6385                };
6386                // A failed agent's edits must not reach the next one, and a
6387                // restore that did not work ends the chain: the call fails
6388                // and the caller's own (confirmed) restore decides.
6389                if let Some((wt, rev)) = restore
6390                    && let Err(e) = self.graft_restore(wt, rev).await
6391                {
6392                    let why =
6393                        format!("could not restore the worktree before the next fixer: {e:#}");
6394                    tracing::warn!("{why}");
6395                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6396                        AgentOutcome::ResourceBlocked(why)
6397                    } else {
6398                        AgentOutcome::Failed(why)
6399                    };
6400                    last = Some((job, seat, out));
6401                    break;
6402                }
6403                record_handover(
6404                    &mut self.state,
6405                    node,
6406                    &seat.key,
6407                    &spec.id,
6408                    next,
6409                    &class,
6410                    &reason,
6411                );
6412                continue;
6413            }
6414            last = Some((job, seat, out));
6415            break;
6416        }
6417        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6418        if let AgentOutcome::Quota(output) = &out
6419            && node != "graft"
6420        {
6421            // Graft is best effort: its caller records the loss and restores
6422            // the pre-graft tree, then proceeds to review and gate.
6423            // Every fallback was tried. Machine outages cannot be repaired by
6424            // spending a fix round; use each caller's resumable resource path.
6425            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6426            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6427            self.state.quota.push(loss);
6428            self.state.seats.insert(seat.key.clone(), seat.clone());
6429            self.state.status = RunStatus::Blocked;
6430            return (
6431                job,
6432                seat,
6433                AgentOutcome::ResourceBlocked(
6434                    "fixer unavailable; restore agent availability and resume".into(),
6435                ),
6436            );
6437        }
6438        (job, seat, out)
6439    }
6440
6441    /// The one repair a verification-only failure earns (see
6442    /// [`crate::recovery`]), before the first review round of a run that took
6443    /// a branch over.
6444    ///
6445    /// Saved as started *before* the fixer is called, so a resume never
6446    /// repairs twice; a start found on re-entry is rolled back to the commit
6447    /// it began from. A failed or unusable fixer leaves the tree as it was and
6448    /// the review goes on over it, so the first round's own verification
6449    /// reports the failure again. The failure is never carried into a round as
6450    /// a result: verification is re-run on whatever commit this leaves.
6451    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6452        use crate::recovery::{Mode, Repair};
6453        if !self.state.reviews.is_empty() {
6454            return Ok(());
6455        }
6456        let Some(mut rec) = self.state.recovery.clone() else {
6457            return Ok(());
6458        };
6459        if rec.mode != Mode::VerifyRepair {
6460            return Ok(());
6461        }
6462        let wt = winner.worktree.clone();
6463        let mut head = git::rev_parse(&wt, "HEAD").await?;
6464        match rec.repair.clone() {
6465            Repair::Done { .. } => return Ok(()),
6466            Repair::WaitingForRestore { before } => {
6467                self.graft_restore(&wt, &before).await?;
6468                head = before;
6469                rec.repair = Repair::NotStarted;
6470                self.state.recovery = Some(rec.clone());
6471                self.state.save()?;
6472            }
6473            Repair::Started { before } => {
6474                // Interrupted: the unfinished edits are not trusted.
6475                // An unconfirmed rollback blocks the run with `Started` kept,
6476                // so a resume retries it; the tree is never reviewed as is.
6477                if let Err(e) = self.graft_restore(&wt, &before).await {
6478                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6479                        return Err(e);
6480                    }
6481                    return self
6482                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6483                        .await;
6484                }
6485                rec.repair = Repair::Done {
6486                    before: before.clone(),
6487                    after: before,
6488                    committed: false,
6489                    failed: Some("interrupted; rolled back".to_owned()),
6490                };
6491                self.state
6492                    .event("recovery", "the repair was interrupted and rolled back");
6493                self.state.recovery = Some(rec);
6494                self.state.save()?;
6495                return Ok(());
6496            }
6497            Repair::NotStarted => {}
6498        }
6499        if crate::recovery::recheck(&mut rec, &head) {
6500            self.state.event("recovery", rec.summary());
6501            self.state.recovery = Some(rec);
6502            self.state.save()?;
6503            return Ok(());
6504        }
6505        rec.repair = Repair::Started {
6506            before: head.clone(),
6507        };
6508        self.state.recovery = Some(rec.clone());
6509        self.state.save()?;
6510
6511        let language = self.state.config.graph.language.clone();
6512        let sessions = self.state.config.graph.sessions;
6513        let artifacts = agent::artifacts_dir(&self.state.dir());
6514        let fix_prompt = prompt::recovery_fix(
6515            &self.state.instruction,
6516            &rec,
6517            self.state.config.graph.review_rounds,
6518            &language,
6519        );
6520        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6521        self.state.event(
6522            "recovery",
6523            format!(
6524                "repair: the fixer starts from the failed verification on {}",
6525                short(&head)
6526            ),
6527        );
6528        let (_job, seat, out) = self
6529            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6530                prompt: fix_prompt.clone(),
6531                spec,
6532                seat,
6533                cwd: wt.clone(),
6534                timeout,
6535                allow_write: true,
6536                sessions,
6537                artifacts: artifacts.clone(),
6538                stem: "fix-recovery".to_owned(),
6539                handover: None,
6540            })
6541            .await;
6542        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6543            rec.repair = Repair::WaitingForRestore { before: head };
6544            self.state.recovery = Some(rec);
6545            return self.stop_for_resource();
6546        }
6547        let failed = match &out {
6548            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6549            AgentOutcome::Quota(o) => {
6550                self.state.quota.push(QuotaLoss::from_report(
6551                    seat.key.clone(),
6552                    "fix",
6553                    o.quota.as_ref(),
6554                ));
6555                Some("rate limited (quota); the fixer could not run".to_owned())
6556            }
6557            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6558            AgentOutcome::Failed(e) => Some(e.clone()),
6559        };
6560        self.state.seats.insert(seat.key.clone(), seat);
6561        let coordination = self.coordinate(&wt).await?;
6562        let failed = match failed {
6563            // The agent's partial edits are not kept.
6564            Some(why) => {
6565                if let Err(e) = git::restore_to(&wt, &head).await {
6566                    return self
6567                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6568                        .await;
6569                }
6570                Some(why)
6571            }
6572            None => {
6573                if let Ok(r) = git::rescue_commit(
6574                    &wt,
6575                    "magi: recovery repair of the failed verification (uncommitted work)",
6576                )
6577                .await
6578                {
6579                    self.state.note_withheld("fix", &r.withheld);
6580                }
6581                None
6582            }
6583        };
6584        let after = git::rev_parse(&wt, "HEAD").await?;
6585        drop(coordination);
6586        self.state.event(
6587            "recovery",
6588            match &failed {
6589                Some(why) => format!("repair: nothing kept ({why})"),
6590                None if after != head => {
6591                    format!("repair: committed {}", short(&after))
6592                }
6593                None => "repair: the fixer made no commit".to_owned(),
6594            },
6595        );
6596        rec.repair = Repair::Done {
6597            before: head.clone(),
6598            committed: after != head,
6599            after,
6600            failed,
6601        };
6602        self.state.recovery = Some(rec);
6603        self.state.save()?;
6604        Ok(())
6605    }
6606
6607    /// A repair whose rollback could not be confirmed: the run is blocked and
6608    /// resumable, and the unconfirmed state is never reviewed.
6609    async fn stop_recovery(
6610        &mut self,
6611        mut rec: crate::recovery::RecoveryRecord,
6612        head: &str,
6613        why: String,
6614    ) -> Result<()> {
6615        // `Started` stays recorded: a resume retries the restore.
6616        rec.repair = crate::recovery::Repair::Started {
6617            before: head.to_owned(),
6618        };
6619        self.state.recovery = Some(rec);
6620        self.state
6621            .event("recovery", format!("repair could not be undone: {why}"));
6622        self.state.status = RunStatus::Blocked;
6623        self.state.save()?;
6624        bail!("recovery repair could not be rolled back: {why}")
6625    }
6626
6627    async fn review_loop(&mut self) -> Result<()> {
6628        if self.hold_base_sync().await? {
6629            return Ok(());
6630        }
6631        loop {
6632            if self
6633                .state
6634                .base_sync
6635                .as_ref()
6636                .is_some_and(|s| s.conflict.is_some())
6637            {
6638                return Ok(());
6639            }
6640            if self.catchup_repairable() {
6641                let record = self.state.reviews.last().expect("repairable round").clone();
6642                let winner = self
6643                    .state
6644                    .winner()
6645                    .cloned()
6646                    .context("catch-up repair has no winner")?;
6647                if !self.fix_review_round(&winner, record).await? {
6648                    return Ok(());
6649                }
6650            }
6651            self.review_loop_pass().await?;
6652            if !self.catchup_repairable() {
6653                return Ok(());
6654            }
6655        }
6656    }
6657
6658    /// Derive an owed catch-up repair from durable round evidence, including
6659    /// reentry after a crash before the fixer was dispatched.
6660    fn catchup_repairable(&self) -> bool {
6661        self.state.reviews.len() < self.state.config.graph.review_rounds
6662            && self.state.reviews.last().is_some_and(|r| {
6663                !r.clean
6664                    && r.blocking == 0
6665                    && r.fix.is_none()
6666                    && r.e2e_status() == E2eStatus::Failed
6667            })
6668    }
6669
6670    async fn review_loop_pass(&mut self) -> Result<()> {
6671        // A base that would not rebase is a person's decision, not a review
6672        // round: nothing here would change the answer, and reviewers and a
6673        // fixer would be spending real budget on a tree that cannot land
6674        // regardless of what they find.
6675        if self
6676            .state
6677            .base_sync
6678            .as_ref()
6679            .is_some_and(|s| s.conflict.is_some())
6680        {
6681            return Ok(());
6682        }
6683        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
6684        // agent files with `magi task add` name the run that paid for it. The
6685        // prompt overlay is cloned alongside it because the waves borrow it
6686        // while `self` is mutably borrowed by the node's own bookkeeping.
6687        let run_id = self.state.id.clone();
6688        let prompts = self.state.config.prompts.clone();
6689        let Some(winner) = self.state.winner().cloned() else {
6690            return Ok(());
6691        };
6692        let granted_review = self
6693            .state
6694            .repair_grants
6695            .last()
6696            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
6697            .cloned();
6698        let fresh_review = granted_review
6699            .as_ref()
6700            .is_some_and(|g| self.state.reviews.len() == g.review_start);
6701        let max_rounds = granted_review
6702            .as_ref()
6703            .map_or(self.state.config.graph.review_rounds, |g| {
6704                g.review_start.saturating_add(1)
6705            });
6706        // A clean round, an exhausted round budget, or a stalled tree (see
6707        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
6708        // they are recorded — recomputed here, not read off `status`, so a
6709        // reentry into a run that already stopped restates the identical
6710        // verdict instead of silently handing back whatever an earlier node
6711        // in this same walk clobbered `status` to (a solo-candidate
6712        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
6713        // below runs an empty range once the budget is spent, and would
6714        // otherwise fall through without touching `status` at all.
6715        if let Some(status) =
6716            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
6717        {
6718            // A reentry after a crash between the last round's save and
6719            // `stop_reviewing` reaches the hand-off here, not there.
6720            if status == RunStatus::Gating {
6721                self.record_contested_handoff();
6722            }
6723            self.state.status = status;
6724            self.state.save()?;
6725            return Ok(());
6726        }
6727        self.state.status = RunStatus::Reviewing;
6728        // A last recorded round whose own verification never resolved
6729        // (`ResourceBlocked` — the shared build cache, not the patch) is
6730        // never a concluded round, whatever the round budget says: starting
6731        // a fresh round on top of it would spend a whole new reviewer wave
6732        // re-reading an unchanged patch instead of just retrying the one
6733        // check that actually needs it, and once the budget is spent the
6734        // loop below has nothing left to do at all (its range is empty).
6735        // Retry that check directly instead, exactly the same retry
6736        // `stop_reviewing` already does for its own catch-up case.
6737        if !fresh_review
6738            && self.state.reviews.last().is_some_and(|r| {
6739                r.e2e_status() == E2eStatus::ResourceBlocked
6740                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6741            })
6742        {
6743            let shell = self.state.config.shell();
6744            return self
6745                .stop_unresolved_review(
6746                    "the last round's own verification never resolved",
6747                    &shell,
6748                    &winner.worktree,
6749                )
6750                .await;
6751        }
6752
6753        if granted_review.is_none() {
6754            self.recovery_repair(&winner).await?;
6755        }
6756        if self.resource_halt()? {
6757            return Ok(());
6758        }
6759
6760        let repo = self.state.repo.clone();
6761        let root = self.state.worktree_root();
6762        let language = self.state.config.graph.language.clone();
6763        let sessions = self.state.config.graph.sessions;
6764        let artifacts = self.merge_repair_artifacts();
6765        let base = self.landing_base();
6766        let base_short = short(&base);
6767        let reviewers = self.roles.reviewers.clone();
6768        let shell = self.state.config.shell();
6769
6770        for round in (self.state.reviews.len() + 1)..=max_rounds {
6771            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6772            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6773            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6774            // The prior round's own record, already persisted — never a
6775            // hand-carried variable of just its failing output: that is
6776            // exactly what let a round's e2e result drift out of sync with
6777            // which commit it was actually about (see `SCHEMA`'s doc for
6778            // schema 8). Judged against `head`, the commit reviewers are
6779            // about to look at now, so the summary always reads as "an
6780            // earlier head" here — this round's own patch has not been
6781            // checked yet.
6782            let prev_verification = self
6783                .state
6784                .reviews
6785                .last()
6786                .and_then(|r| r.verification_summary(&head));
6787            // Only the first round of a run that took a branch over: later
6788            // rounds have this run's own history.
6789            let prior_section = match (&self.state.recovery, round) {
6790                (Some(rec), 1) if self.state.reviews.is_empty() => {
6791                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6792                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6793                            Some(crate::recovery::head_of(
6794                                &git::diff(&winner.worktree, reviewed, "HEAD")
6795                                    .await
6796                                    .unwrap_or_default(),
6797                                crate::recovery::SINCE_DIFF_MAX,
6798                            ))
6799                        }
6800                        _ => None,
6801                    };
6802                    Some(prompt::recovery_review(rec, since.as_deref()))
6803                }
6804                _ => None,
6805            };
6806
6807            // Each reviewer gets its own detached checkout of exactly this
6808            // commit: nobody can perturb the winner's tree, and the fixer can
6809            // keep working without racing a reviewer.
6810            let mut jobs = Vec::new();
6811            let coordination = self.coordinate(&repo).await?;
6812            for (r, spec) in reviewers.iter().cloned().enumerate() {
6813                let wt = root.join(format!("review-{}", r + 1));
6814                // `clean -fdx` in a reset wipes setup's products, so every
6815                // round sets the seat up again.
6816                if wt.exists() {
6817                    git::reset_detached(&wt, &head).await?;
6818                } else {
6819                    git::worktree_add_detached(&repo, &wt, &head).await?;
6820                }
6821                self.setup_seat_worktree(&repo, &wt, false).await?;
6822                let seat_key = format!("review-{}", r + 1);
6823                // The seat starts the round on whoever answered it last, not
6824                // on the agent the spec names, so a failure is not re-paid.
6825                let spec = pick_start_spec(
6826                    &self.roles.reviewer_roster,
6827                    spec,
6828                    self.state.seat_history.get(&seat_key),
6829                );
6830                let seat = self.seat(&seat_key, &spec.id);
6831                jobs.push(SeatJob {
6832                    prompt: prompt::review(&prompt::ReviewCtx {
6833                        instruction: &self.state.instruction,
6834                        branch: &winner.branch,
6835                        base_short: &base_short,
6836                        stat: &stat,
6837                        patch: &patch,
6838                        verification: prev_verification.as_ref(),
6839                        prior: prior_section.as_deref(),
6840                        reviewers: reviewers.len(),
6841                        round,
6842                        rounds: max_rounds,
6843                        // A review-only run has no rankings, so nothing
6844                        // competed for this patch and the reviewer is told so.
6845                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6846                        lens: Lens::for_seat(r),
6847                        language: &language,
6848                        github_language: &self.state.config.graph.github_language,
6849                    }),
6850                    spec,
6851                    seat,
6852                    cwd: wt,
6853                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6854                    allow_write: false,
6855                    sessions,
6856                    artifacts: artifacts.clone(),
6857                    stem: format!("review-{round}-{}", r + 1),
6858                    handover: None,
6859                });
6860            }
6861
6862            drop(coordination);
6863            self.state.event(
6864                "review",
6865                format!(
6866                    "round {round}: {} reviewers on {}",
6867                    jobs.len(),
6868                    short(&head)
6869                ),
6870            );
6871            let mut quota_losses = Vec::new();
6872            let review_retries = self.state.config.graph.retries;
6873            let review_cache = self.state.config.cache_dir();
6874            let ctx = WaveCtx {
6875                carry_seats: true,
6876                pauses: Some((&self.pause, &self.interrupt)),
6877                run: &run_id,
6878                node: "review",
6879                prompts: &prompts,
6880                cache: review_cache.as_deref(),
6881                round: Some(round),
6882            };
6883            let results = ask_json_wave::<Review>(
6884                jobs,
6885                Arc::clone(&self.sem),
6886                review_retries,
6887                &self.roles.reviewer_roster,
6888                &ctx,
6889                &mut quota_losses,
6890                &mut self.state,
6891                &|_: &Review| Ok(()),
6892            )
6893            .await;
6894            if self.state.resource_block.is_some() {
6895                return self.stop_for_resource();
6896            }
6897            // Counted before the move below: how many of *this* round's
6898            // reviewer seats were lost to their own rate limit, as opposed to
6899            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6900            let round_quota_missing = quota_losses.len();
6901            self.state.quota.extend(quota_losses);
6902
6903            let mut records = Vec::new();
6904            let mut all_findings = Vec::new();
6905            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6906                let agent_id = seat.agent.clone();
6907                self.state.seats.insert(seat.key.clone(), seat);
6908                let mut record = ReviewRecord {
6909                    reviewer: r + 1,
6910                    agent: agent_id,
6911                    summary: String::new(),
6912                    findings: Vec::new(),
6913                    vote: None,
6914                    failed: None,
6915                    duration_ms: 0,
6916                    // Set for both outcomes: `failed: Some(_)` with
6917                    // `attempts > 0` is a seat every retry still lost, not a
6918                    // recovered one — only `failed: None` with `attempts > 0`
6919                    // reads as "answered after a nudge" (see this field's own
6920                    // doc).
6921                    attempts,
6922                };
6923                match res {
6924                    Ok((review, out)) => {
6925                        // Sanitized here, at the point every other piece of
6926                        // agent prose in this file is (candidate summaries,
6927                        // deliberation turns, vote reasons): a reviewer's own
6928                        // words are the one thing about it that could name
6929                        // it, and reconsideration below broadcasts this same
6930                        // summary and these same findings to every other
6931                        // seat on the panel.
6932                        record.summary =
6933                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6934                        record.vote = Some(review.vote);
6935                        record.duration_ms = out.duration_ms;
6936                        for (n, mut f) in review.findings.into_iter().enumerate() {
6937                            // ids are magi's, never the agent's: the fixer's
6938                            // adoption report is keyed by them.
6939                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6940                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6941                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6942                            // `file` is agent-supplied prose too, never
6943                            // checked against the real tree — the same
6944                            // exposure `title`/`detail` above have, just in
6945                            // a field easy to forget because it looks like a
6946                            // path rather than free text.
6947                            f.file = f
6948                                .file
6949                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6950                            all_findings.push(f.clone());
6951                            record.findings.push(f);
6952                        }
6953                        self.state.event(
6954                            "review",
6955                            format!(
6956                                "round {round}: reviewer {} voted {} with {} finding(s)",
6957                                r + 1,
6958                                review.vote.label(),
6959                                record.findings.len()
6960                            ),
6961                        );
6962                    }
6963                    Err(e) => {
6964                        record.failed = Some(e.to_string());
6965                        self.state.event(
6966                            "review",
6967                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6968                        );
6969                    }
6970                }
6971                records.push(record);
6972            }
6973
6974            // Tally the round's votes and, if they split, spend the one
6975            // round of reconsideration the split -> deliberate -> revote
6976            // shape `judge`/`vote` use for the panel, sized down to what a
6977            // read-only review round can afford: one round, and a revote
6978            // rather than an argument, because the panel already wrote its
6979            // reasoning down as findings the first time around.
6980            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6981            let vote_split =
6982                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6983            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6984            if vote_split {
6985                self.state.event(
6986                    "review",
6987                    format!(
6988                        "round {round}: votes split ({}) — one round of reconsideration",
6989                        initial_votes
6990                            .iter()
6991                            .map(|v| v.label())
6992                            .collect::<Vec<_>>()
6993                            .join(", ")
6994                    ),
6995                );
6996                // Seats read every seat's findings and votes, still numbered
6997                // and never named — the same anonymity `review` itself keeps.
6998                let panel: Vec<ReviewSeatReport<'_>> = records
6999                    .iter()
7000                    .filter_map(|r| {
7001                        r.vote.map(|vote| ReviewSeatReport {
7002                            reviewer: r.reviewer,
7003                            vote,
7004                            summary: &r.summary,
7005                            findings: &r.findings,
7006                        })
7007                    })
7008                    .collect();
7009
7010                let mut jobs = Vec::new();
7011                let mut seats_at = Vec::new();
7012                for (r, spec) in reviewers.iter().cloned().enumerate() {
7013                    // A seat with no initial vote has nothing to reconsider
7014                    // from and stays absent, the same as it stayed absent
7015                    // from `panel` above.
7016                    if records[r].vote.is_none() {
7017                        continue;
7018                    }
7019                    let wt = root.join(format!("review-{}", r + 1));
7020                    let seat_key = format!("review-{}", r + 1);
7021                    let spec = self.occupant(&seat_key, spec);
7022                    let seat = self.seat(&seat_key, &spec.id);
7023                    // A seat with no live session has already forgotten the
7024                    // initial review's prompt — restate the patch it is
7025                    // voting on, the same as `deliberate`/`vote` do for a
7026                    // judge in the same position.
7027                    // The panel already carries this seat's own review and
7028                    // vote, so restating the patch makes the prompt whole for
7029                    // a seat handed to another agent.
7030                    let build = |with_patch: bool| {
7031                        prompt::review_reconsider(&ReviewReconsiderCtx {
7032                            instruction: &self.state.instruction,
7033                            reviewer: r + 1,
7034                            lens: Lens::for_seat(r),
7035                            panel: &panel,
7036                            patch: with_patch.then_some(ReviewPatch {
7037                                branch: &winner.branch,
7038                                base_short: &base_short,
7039                                stat: &stat,
7040                                patch: &patch,
7041                            }),
7042                            round,
7043                            rounds: max_rounds,
7044                            language: &language,
7045                        })
7046                    };
7047                    let full = build(true);
7048                    let prompt = if has_context(&spec, &seat, sessions) {
7049                        build(false)
7050                    } else {
7051                        full.clone()
7052                    };
7053                    jobs.push(SeatJob {
7054                        prompt,
7055                        spec,
7056                        seat,
7057                        cwd: wt,
7058                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7059                        allow_write: false,
7060                        sessions,
7061                        artifacts: artifacts.clone(),
7062                        stem: format!("review-{round}-reconsider-{}", r + 1),
7063                        handover: Some(full),
7064                    });
7065                    seats_at.push(r);
7066                }
7067
7068                let mut recon_quota_losses = Vec::new();
7069                let recon_cache = self.state.config.cache_dir();
7070                let recon_ctx = WaveCtx {
7071                    carry_seats: true,
7072                    pauses: Some((&self.pause, &self.interrupt)),
7073                    run: &run_id,
7074                    node: "review",
7075                    prompts: &prompts,
7076                    cache: recon_cache.as_deref(),
7077                    round: Some(round),
7078                };
7079                let recon_results = ask_json_wave::<ReviewRevote>(
7080                    jobs,
7081                    Arc::clone(&self.sem),
7082                    review_retries,
7083                    &self.roles.reviewer_roster,
7084                    &recon_ctx,
7085                    &mut recon_quota_losses,
7086                    &mut self.state,
7087                    &|_: &ReviewRevote| Ok(()),
7088                )
7089                .await;
7090                if self.state.resource_block.is_some() {
7091                    return self.stop_for_resource();
7092                }
7093                self.state.quota.extend(recon_quota_losses);
7094
7095                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7096                    let agent_id = seat.agent.clone();
7097                    self.state.seats.insert(seat.key.clone(), seat);
7098                    let mut rec = ReviewRevoteRecord {
7099                        reviewer: r + 1,
7100                        agent: agent_id,
7101                        vote: None,
7102                        reason: String::new(),
7103                        failed: None,
7104                    };
7105                    match res {
7106                        Ok((rv, _)) => {
7107                            rec.vote = Some(rv.vote);
7108                            rec.reason =
7109                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7110                            self.state.event(
7111                                "review",
7112                                format!(
7113                                    "round {round}: reviewer {} revoted {}",
7114                                    r + 1,
7115                                    rv.vote.label()
7116                                ),
7117                            );
7118                        }
7119                        Err(e) => {
7120                            rec.failed = Some(e.to_string());
7121                            self.state.event(
7122                                "review",
7123                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7124                            );
7125                        }
7126                    }
7127                    reconsideration.push(rec);
7128                }
7129            } else if initial_votes.len() > 1 {
7130                self.state.event(
7131                    "review",
7132                    format!(
7133                        "round {round}: votes agreed ({}) — no reconsideration",
7134                        initial_votes[0].label()
7135                    ),
7136                );
7137            }
7138
7139            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7140            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7141            // A round that already has a blocking finding and a round left to
7142            // try is going back to the fixer no matter what `verify.e2e`
7143            // says, so running it first only spends the loop's slowest step
7144            // (minutes, for a Rust repo's full test suite) on a head about
7145            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7146            // runs once a round has no blocking findings left (see
7147            // `round_is_clean`, which a deferred — empty — `e2e` can never
7148            // satisfy since `blocking` is nonzero whenever this branch is
7149            // taken), and `stop_reviewing` forces a real run before it will
7150            // ever read a deferred round as green.
7151            let defer_e2e =
7152                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7153            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7154                let reason =
7155                    format!("{blocking} blocking finding(s) already required a fix this round");
7156                self.state.event(
7157                    "verify",
7158                    format!(
7159                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7160                         {}); it will run once a round has none left",
7161                        short(&head)
7162                    ),
7163                );
7164                (Vec::new(), false, true, Some(reason))
7165            } else {
7166                let e2e_commands = self.state.config.verify.e2e.clone();
7167                let cache_dir = self.state.config.cache_dir();
7168                let context = format!("round {round}");
7169                let wait = self.repository_wait();
7170                let (e2e, verify_retried) = with_cache_lease(
7171                    &mut self.state,
7172                    cache_dir.as_deref(),
7173                    "e2e",
7174                    "e2e",
7175                    &winner.worktree,
7176                    &head,
7177                    verify_timeout,
7178                    &context,
7179                    &wait,
7180                    |state, budget| {
7181                        let shell = shell.clone();
7182                        let e2e_commands = e2e_commands.clone();
7183                        let worktree = winner.worktree.clone();
7184                        let context = context.clone();
7185                        async move {
7186                            run_e2e_with_retry(
7187                                state,
7188                                &shell,
7189                                &e2e_commands,
7190                                &worktree,
7191                                budget,
7192                                &context,
7193                            )
7194                            .await
7195                        }
7196                    },
7197                )
7198                .await?;
7199                (e2e, verify_retried, false, None)
7200            };
7201
7202            let expected = records.len();
7203            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7204            let incomplete = answered < expected;
7205            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7206            let policy = self.state.config.graph.incomplete_review;
7207            let clean = round_is_clean(
7208                blocking,
7209                e2e_ok,
7210                answered,
7211                expected,
7212                round_quota_missing,
7213                policy,
7214            );
7215
7216            let mut round_record = ReviewRound {
7217                round,
7218                head: head.clone(),
7219                verified_head: None,
7220                verified_at: None,
7221                reviews: records,
7222                e2e,
7223                verify_retried,
7224                e2e_deferred,
7225                e2e_defer_reason,
7226                fix: None,
7227                blocking,
7228                answered,
7229                expected,
7230                clean,
7231                progressed: false,
7232                vote_split,
7233                reconsideration,
7234                verdict: None,
7235            };
7236            // The final vote per seat is its revote where reconsideration
7237            // ran and answered, its initial vote otherwise — the same
7238            // fallback `tally` uses for a judge whose private vote failed.
7239            round_record.verdict = ReviewVote::worst(
7240                round_record
7241                    .final_votes()
7242                    .into_iter()
7243                    .map(|(_, _, vote)| vote),
7244            );
7245            // Which commit and when magi actually attempted to check —
7246            // known the moment a command was dispatched against `head`,
7247            // whether or not it finished: a resource-blocked attempt still
7248            // targeted a specific commit at a specific time, and leaving
7249            // that unrecorded is exactly what made `verification_summary`
7250            // report a fresh attempt as "commit unknown ... recorded before
7251            // this was tracked", indistinguishable from a genuinely old,
7252            // untracked record. Only a deferred or unconfigured round never
7253            // ran at all and has nothing to record — see
7254            // `ReviewRound::verified_head`'s own doc.
7255            if !matches!(
7256                round_record.e2e_status(),
7257                E2eStatus::Deferred | E2eStatus::NotConfigured
7258            ) {
7259                round_record.verified_head = Some(head.clone());
7260                round_record.verified_at = Some(Timestamp::now());
7261            }
7262
7263            if incomplete {
7264                let missing: Vec<String> = round_record
7265                    .reviews
7266                    .iter()
7267                    .filter(|r| r.failed.is_some())
7268                    .map(|r| format!("review-{}", r.reviewer))
7269                    .collect();
7270                self.state.event(
7271                    "review",
7272                    format!(
7273                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7274                        missing.join(", ")
7275                    ),
7276                );
7277            }
7278
7279            if clean {
7280                self.state.event(
7281                    "review",
7282                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7283                        format!(
7284                            "round {round}: clean (warn policy, incomplete panel) — no \
7285                             blocking findings from the seats that answered, verification green"
7286                        )
7287                    } else if incomplete {
7288                        format!(
7289                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7290                             quorum) — no blocking findings from the seats that answered, \
7291                             verification green",
7292                            expected - answered
7293                        )
7294                    } else {
7295                        format!("round {round}: clean — no blocking findings, verification green")
7296                    },
7297                );
7298                self.state.reviews.push(round_record);
7299                self.state.status = RunStatus::Gating;
7300                self.state.save()?;
7301                return Ok(());
7302            }
7303
7304            // Nothing was raised and verification passed, but not every seat
7305            // answered and `round_is_clean` still refused to call it clean —
7306            // either a seat is missing for a reason other than its own quota
7307            // (a crash, a timeout, unparsable output — worth another try), or
7308            // every seat that could have answered lost its quota and nobody
7309            // is left to decide on: re-review rather than send the fixer
7310            // after a round with nothing to fix.
7311            if incomplete && blocking == 0 && e2e_ok {
7312                self.state.reviews.push(round_record);
7313                self.state.save()?;
7314                if round == max_rounds {
7315                    self.state.status = RunStatus::Blocked;
7316                    self.state.event(
7317                        "review",
7318                        format!(
7319                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7320                             refusing to call it clean",
7321                            expected - answered
7322                        ),
7323                    );
7324                    return Ok(());
7325                }
7326                continue;
7327            }
7328
7329            // Nothing for the fixer to act on (`blocking == 0`) and the only
7330            // reason this round is not clean is that magi itself never got
7331            // a command to run — the shared build cache, not the patch (see
7332            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7333            // the fixer would invite a change to appease contention that has
7334            // nothing to do with the diff, and would leave this attempt
7335            // sitting in the next round's prompt as if it were about an
7336            // earlier, superseded commit rather than what it actually is:
7337            // the same head, still waiting to be checked. Wait for it the
7338            // same way the final round's own contention is already handled,
7339            // whatever round this happens to be.
7340            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7341                self.state.reviews.push(round_record);
7342                return self
7343                    .stop_unresolved_review(
7344                        "the round's own verification could not run",
7345                        &shell,
7346                        &winner.worktree,
7347                    )
7348                    .await;
7349            }
7350
7351            if round == max_rounds {
7352                self.state.reviews.push(round_record);
7353                return self
7354                    .stop_reviewing(
7355                        &format!(
7356                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7357                        ),
7358                        &shell,
7359                        &winner.worktree,
7360                    )
7361                    .await;
7362            }
7363
7364            if !self.fix_review_round(&winner, round_record).await? {
7365                return Ok(());
7366            }
7367        }
7368        Ok(())
7369    }
7370
7371    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7372    /// False means admission or continuation policy stopped the review loop.
7373    async fn fix_review_round(
7374        &mut self,
7375        winner: &Candidate,
7376        mut round_record: ReviewRound,
7377    ) -> Result<bool> {
7378        let round = round_record.round;
7379        let shell = self.state.config.shell();
7380        let step = self.run_fix_round(winner, &mut round_record).await?;
7381        if matches!(step, FixStep::Resource) {
7382            self.stop_for_resource()?;
7383            return Ok(false);
7384        }
7385        if self.state.reviews.len() < round {
7386            self.state.reviews.push(round_record);
7387        } else {
7388            self.state.reviews[round - 1] = round_record;
7389        }
7390        self.state.save()?;
7391
7392        // The fixer's own report never came back this round, even after
7393        // `continue_fix_report`'s own budget was spent on it — not an
7394        // ordinary "no report" (dropped stream, quota, plain failure),
7395        // which already reads that way and is left to the existing round
7396        // budget. Stopping here, rather than opening another round, is
7397        // what keeps a next reviewer/fixer wave from ever being
7398        // dispatched onto `winner.worktree` while whatever the seat's
7399        // last call may still have running there is unaccounted for: no
7400        // process liveness check exists (and none is being added — see
7401        // AGENTS.md/this task's own scope), so the only way to honour
7402        // "nothing starts before a valid report returns" is to not start
7403        // anything further on this worktree from this run at all.
7404        if matches!(step, FixStep::Recorded { report_lost: true }) {
7405            self.stop_reviewing(
7406                "the fixer's adoption report never came back, even after resuming its \
7407                         own seat; refusing to start another round against the same worktree \
7408                         while that is unresolved",
7409                &shell,
7410                &winner.worktree,
7411            )
7412            .await?;
7413            return Ok(false);
7414        }
7415
7416        let streak = self
7417            .state
7418            .reviews
7419            .iter()
7420            .rev()
7421            .take_while(|r| !r.progressed)
7422            .count();
7423        if streak >= STAGNANT_LIMIT {
7424            self.stop_reviewing(
7425                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7426                &shell,
7427                &winner.worktree,
7428            )
7429            .await?;
7430            return Ok(false);
7431        }
7432        Ok(true)
7433    }
7434
7435    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7436    async fn run_fix_round(
7437        &mut self,
7438        winner: &Candidate,
7439        round_record: &mut ReviewRound,
7440    ) -> Result<FixStep> {
7441        let round = round_record.round;
7442        let max_rounds = self.state.config.graph.review_rounds;
7443        let run_id = self.state.id.clone();
7444        let prompts = self.state.config.prompts.clone();
7445        let language = self.state.config.graph.language.clone();
7446        let sessions = self.state.config.graph.sessions;
7447        let artifacts = self.merge_repair_artifacts();
7448        let base = self.landing_base();
7449        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7450        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7451        let this_round_verification = round_record.verification_summary(&head);
7452        let all_findings: Vec<_> = round_record
7453            .reviews
7454            .iter()
7455            .flat_map(|r| r.findings.iter().cloned())
7456            .collect();
7457        // Fix. The winner's own implementer seat continues its conversation:
7458        // the competition is over, so context is pure benefit now.
7459        let blocking_findings: Vec<_> = all_findings
7460            .iter()
7461            .filter(|f| f.severity.blocks())
7462            .cloned()
7463            .collect();
7464        let fix_prompt = prompt::fix_in(
7465            &self.state.instruction,
7466            &blocking_findings,
7467            this_round_verification.as_ref(),
7468            round,
7469            max_rounds,
7470            &language,
7471            &self.state.config.graph.github_language,
7472        );
7473        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7474        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7475        let (job, seat, out) = self
7476            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7477                prompt: fix_prompt.clone(),
7478                spec,
7479                seat,
7480                cwd: winner.worktree.clone(),
7481                timeout,
7482                allow_write: true,
7483                sessions,
7484                artifacts: artifacts.clone(),
7485                stem: format!("fix-{round}"),
7486                handover: None,
7487            })
7488            .await;
7489        // Not asked: the round is not recorded (so a resume repeats it),
7490        // no fixer was handed over to, and the seat is left as it was.
7491        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7492            return Ok(FixStep::Resource);
7493        }
7494        let agent_id = seat.agent.clone();
7495
7496        let mut fix = FixRecord {
7497            agent: agent_id,
7498            addressed: Vec::new(),
7499            rejected: Vec::new(),
7500            notes: String::new(),
7501            committed: false,
7502            failed: None,
7503            duration_ms: 0,
7504            continuation: None,
7505        };
7506        let mut continuation = ContinuationRecord::not_needed();
7507        let mut final_seat = seat.clone();
7508        match out {
7509            AgentOutcome::Ok(o) => {
7510                fix.duration_ms = o.duration_ms;
7511                let parsed = verdict::extract_json::<FixReport>(&o.text);
7512                // A parsed report standing next to a command this same
7513                // reply's own CLI never confirmed the exit status of is
7514                // not a resolved answer — the identical `CommandEvidence`
7515                // `state.jobs` renders, read here instead of only on
7516                // display, per the completion judgment and the shown
7517                // record needing to agree.
7518                let incomplete_reason = match &parsed {
7519                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7520                        "the reply parsed, but it reported a command whose own CLI never \
7521                             confirmed an exit status"
7522                            .to_owned(),
7523                    ),
7524                    Ok(_) => None,
7525                    Err(e) => Some(e.to_string()),
7526                };
7527                match incomplete_reason {
7528                    None => {
7529                        let report = parsed.expect("checked Ok above");
7530                        fix.addressed = report.addressed;
7531                        fix.rejected = report.rejected;
7532                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7533                    }
7534                    Some(reason) => {
7535                        let (resumed_seat, resolved, failure, cont) = self
7536                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7537                            .await;
7538                        fix.duration_ms += cont.cumulative_wait_ms;
7539                        continuation = cont;
7540                        final_seat = resumed_seat;
7541                        match resolved {
7542                            Some(report) => {
7543                                fix.addressed = report.addressed;
7544                                fix.rejected = report.rejected;
7545                                fix.notes =
7546                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
7547                            }
7548                            None => fix.failed = failure,
7549                        }
7550                    }
7551                }
7552            }
7553            // The CLI's raw error JSON is not a fix report to parse.
7554            AgentOutcome::Dropped(o) => {
7555                fix.duration_ms = o.duration_ms;
7556                let why = o
7557                    .dropped
7558                    .as_ref()
7559                    .map(|d| d.why.as_str())
7560                    .unwrap_or("the CLI ended the stream without delivering its answer");
7561                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7562            }
7563            AgentOutcome::Quota(o) => {
7564                self.state.quota.push(QuotaLoss::from_report(
7565                    final_seat.key.clone(),
7566                    "fix",
7567                    o.quota.as_ref(),
7568                ));
7569                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7570            }
7571            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7572        }
7573        if self.state.resource_block.is_some() {
7574            self.state.seats.insert(final_seat.key.clone(), final_seat);
7575            return Ok(FixStep::Resource);
7576        }
7577        fix.continuation = Some(continuation);
7578        self.state.seats.insert(final_seat.key.clone(), final_seat);
7579        let coordination = self.coordinate(&winner.worktree).await?;
7580        if let Ok(r) = git::rescue_commit(
7581            &winner.worktree,
7582            &format!("magi: review round {round} fixes (uncommitted work)"),
7583        )
7584        .await
7585        {
7586            self.state.note_withheld("fix", &r.withheld);
7587        }
7588        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7589        fix.committed = after != before;
7590        // Judged by what `git` says moved against base, never by the
7591        // fixer's own `addressed`/`rejected` count — see
7592        // `ReviewRound::progressed`. Propagated with `?`, the same as the
7593        // `patch` snapshot above: swallowing this error would default
7594        // `diff_after` to empty, which almost always differs from a
7595        // non-empty `patch` and reads as "progressed" — exactly backwards
7596        // for a `git` failure the stagnation check cannot see through.
7597        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7598        drop(coordination);
7599        let progressed = diff_after != patch;
7600        let commit_note = if fix.committed {
7601            "committed"
7602        } else {
7603            "NO new commit"
7604        };
7605        let tree_note = if progressed {
7606            "changed vs base"
7607        } else {
7608            "unchanged vs base"
7609        };
7610        self.state.event(
7611            "fix",
7612            match &fix.failed {
7613                // Distinct on purpose from "0 addressed, 0 rejected": the
7614                // fixer's own diff still landed (blocking counts do keep
7615                // falling round over round), only its adoption report did
7616                // not come back, so this must never read like every
7617                // finding was reviewed and declined.
7618                Some(reason) => {
7619                    format!(
7620                        "round {round}: fixer's adoption report was lost ({reason}); \
7621                             {commit_note}, tree {tree_note}"
7622                    )
7623                }
7624                None => format!(
7625                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7626                         {tree_note}{}",
7627                    fix.addressed.len(),
7628                    fix.rejected.len(),
7629                    if continuation.outcome == ContinuationOutcome::Resumed {
7630                        format!(
7631                            " (adoption report recovered after {} continuation(s))",
7632                            continuation.attempts
7633                        )
7634                    } else {
7635                        String::new()
7636                    },
7637                ),
7638            },
7639        );
7640        round_record.fix = Some(fix);
7641        round_record.progressed = progressed;
7642        Ok(FixStep::Recorded {
7643            report_lost: matches!(
7644                continuation.outcome,
7645                ContinuationOutcome::Exhausted
7646                    | ContinuationOutcome::QuotaLost
7647                    | ContinuationOutcome::NoSession
7648            ),
7649        })
7650    }
7651
7652    /// Decide, from the last recorded round's own verification, whether
7653    /// stopping the review loop is a hand-off or a genuine block.
7654    ///
7655    /// Called once the loop has given up trying — the round budget is spent,
7656    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
7657    /// findings still open, never while a round is still clean or the
7658    /// incomplete-panel case handled inline above. Gate and e2e are facts
7659    /// about the tree; a lingering review finding is an opinion, and this
7660    /// workload's own `magi stats` puts reviewer precision low enough
7661    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
7662    /// must not by itself stand between a green, verified change and the
7663    /// human who decides what to do with it. A red e2e is not an opinion, so
7664    /// that case still blocks, with the failing command and a tail of its
7665    /// output recorded here rather than left in `run.json` for someone to go
7666    /// find.
7667    ///
7668    /// A round that deferred its own e2e (see [`Config::graph`]'s
7669    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
7670    /// only because nothing ran, and treating an empty list as a passing one
7671    /// here is exactly the "deferred painted green" bug this function exists
7672    /// to not have. When the last round's own verification never resolved —
7673    /// deferred on purpose, or a real attempt the shared build cache blocked
7674    /// — this makes (or retries) the real run, on the actual worktree this
7675    /// loop is about to stop touching, before deciding anything. A
7676    /// resource-blocked attempt is likewise never read as either green or
7677    /// red: it is evidence about the machine, not the patch (see
7678    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
7679    /// blocked cache leaves this call without deciding rather than guessing
7680    /// — the caller retries on a later reentry.
7681    /// Record, once, that the review loop handed off over a blocking finding
7682    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
7683    /// `land` asks the owner even with `land_approval` off. Called from every
7684    /// path that concludes `Gating`; a reentry keeps the first record.
7685    fn record_contested_handoff(&mut self) {
7686        if self.state.contested_handoff.is_some() {
7687            return;
7688        }
7689        let Some(contested) = self
7690            .state
7691            .reviews
7692            .last()
7693            .and_then(ReviewRound::contested_handoff)
7694        else {
7695            return;
7696        };
7697        self.state.event(
7698            "review",
7699            format!(
7700                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7701                 wait for the owner's approval",
7702                contested.findings.len(),
7703                contested.rejecters.len()
7704            ),
7705        );
7706        self.state.contested_handoff = Some(contested);
7707    }
7708
7709    async fn stop_unresolved_review(
7710        &mut self,
7711        why: &str,
7712        shell: &[String],
7713        worktree: &Path,
7714    ) -> Result<()> {
7715        self.stop_reviewing_inner(why, shell, worktree, true).await
7716    }
7717
7718    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7719        self.stop_reviewing_inner(why, shell, worktree, false).await
7720    }
7721
7722    async fn stop_reviewing_inner(
7723        &mut self,
7724        why: &str,
7725        shell: &[String],
7726        worktree: &Path,
7727        repairable: bool,
7728    ) -> Result<()> {
7729        let round_idx = self.state.reviews.len() - 1;
7730        // A deferred round and a resource-blocked one are the same shape
7731        // here: neither has a real result yet, and both get one more
7732        // attempt. Read off `e2e_status` — the single source for this —
7733        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
7734        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
7735        // still retries instead of being read as a settled result the
7736        // instant it stops being empty.
7737        let needs_catchup_run = matches!(
7738            self.state.reviews[round_idx].e2e_status(),
7739            E2eStatus::Deferred | E2eStatus::ResourceBlocked
7740        );
7741        if needs_catchup_run {
7742            let round = self.state.reviews[round_idx].round;
7743            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7744            let commands = self.state.config.verify.e2e.clone();
7745            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7746            let cache_dir = self.state.config.cache_dir();
7747            let context = format!(
7748                "round {round}: verification unresolved, catching up before the final decision"
7749            );
7750            let wait = self.repository_wait();
7751            let (outcomes, verify_retried) = with_cache_lease(
7752                &mut self.state,
7753                cache_dir.as_deref(),
7754                "e2e",
7755                "e2e",
7756                worktree,
7757                &attempted_head,
7758                timeout,
7759                &context,
7760                &wait,
7761                |state, budget| {
7762                    let shell = shell.to_vec();
7763                    let commands = commands.clone();
7764                    let context = context.clone();
7765                    async move {
7766                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7767                            .await
7768                    }
7769                },
7770            )
7771            .await?;
7772            let last = &mut self.state.reviews[round_idx];
7773            last.e2e = outcomes;
7774            last.verify_retried = verify_retried;
7775            // Always the commit and time this attempt actually targeted,
7776            // whether or not it happens to equal the reviewed `head` and
7777            // whether or not a command finished — see
7778            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
7779            // attempt is recorded too, so a later reader sees "attempted
7780            // again at T2" rather than silence.
7781            last.verified_head = Some(attempted_head);
7782            last.verified_at = Some(Timestamp::now());
7783            if verify_inconclusive(&last.e2e) {
7784                // Still not a real result: `e2e_deferred` is left exactly
7785                // as it was, so `needs_catchup_run` above reads
7786                // `ResourceBlocked` (via `e2e_status`, which checks
7787                // `resource_blocked` before `e2e_deferred`) and retries
7788                // again on the next reentry, rather than recording
7789                // contention as a red e2e and blocking the run on it.
7790                self.state.save()?;
7791                return Ok(());
7792            }
7793            last.e2e_deferred = false;
7794            // A complete panel may have been saved as unclean solely because
7795            // verification could not run. Refresh that verdict with the new
7796            // outcomes before a granted repair consumes the round's result.
7797            // Incomplete panels retain their original policy/quota decision.
7798            if last.answered == last.expected {
7799                last.clean = round_is_clean(
7800                    last.blocking,
7801                    last.e2e.iter().all(CommandOutcome::ok),
7802                    last.answered,
7803                    last.expected,
7804                    0,
7805                    self.state.config.graph.incomplete_review,
7806                );
7807            }
7808        }
7809        let last = &self.state.reviews[round_idx];
7810        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7811
7812        match last.e2e_status() {
7813            E2eStatus::Failed => {
7814                let red: Vec<String> = last
7815                    .e2e
7816                    .iter()
7817                    .filter(|o| !o.ok())
7818                    .map(|o| {
7819                        format!(
7820                            "`{}` -> {:?}\n{}",
7821                            o.command,
7822                            o.code,
7823                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7824                        )
7825                    })
7826                    .collect();
7827                let can_repair = last.blocking == 0 && last.fix.is_none();
7828                self.state.reviews[round_idx].clean = false;
7829                self.state
7830                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7831                self.state.status = RunStatus::Blocked;
7832                if repairable
7833                    && can_repair
7834                    && self.state.reviews.len() < self.state.config.graph.review_rounds
7835                {
7836                    self.state.status = RunStatus::Reviewing;
7837                }
7838            }
7839            // `needs_catchup_run` above already retried once this call; if
7840            // it is still blocked, this is magi's own admission it could
7841            // not get a command to run, never a verdict on the patch — the
7842            // run is left exactly where a later reentry can retry again.
7843            E2eStatus::ResourceBlocked => {
7844                self.state.event(
7845                    "review",
7846                    format!(
7847                        "{why}; e2e could not run (resource unavailable); not \
7848                         deciding yet"
7849                    ),
7850                );
7851            }
7852            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7853                self.state.event(
7854                    "review",
7855                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7856                );
7857                self.record_contested_handoff();
7858                self.state.status = RunStatus::Gating;
7859            }
7860        }
7861        self.state.save()?;
7862        Ok(())
7863    }
7864
7865    // ----------------------------------------------------------------- gate
7866
7867    async fn gate(&mut self) -> Result<()> {
7868        if self.hold_base_sync().await? {
7869            return Ok(());
7870        }
7871        // Judged by the review record itself, not by `status`: a solo
7872        // candidate's `judge`/`deliberate` skip rewrites `status` on every
7873        // reentry (see `judge`), and trusting it here is exactly how a run
7874        // that exhausted its review budget got gated and merged a second
7875        // time around. `review_conclusion` recomputes the review loop's own
7876        // verdict from the round records themselves — `Gating` for a clean
7877        // round or a hand-off (see `stop_reviewing`), anything else means the
7878        // loop is still going or genuinely blocked.
7879        // A base the winner could not be replayed onto is a decision, not a
7880        // round: there is no landing tree to gate. Read as its own record for
7881        // the same reason the review verdict is.
7882        if self.state.status == RunStatus::Failed
7883            || self
7884                .state
7885                .base_sync
7886                .as_ref()
7887                .is_some_and(|s| s.conflict.is_some())
7888            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7889                != Some(RunStatus::Gating)
7890        {
7891            return Ok(());
7892        }
7893        let Some(winner) = self.state.winner().cloned() else {
7894            return Ok(());
7895        };
7896        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7897        self.state.reconcile_verification_head(&head);
7898        self.state.save()?;
7899        let wait = self.repository_wait();
7900        if !verify_current_head(&mut self.state, &winner, &wait).await? {
7901            return Ok(());
7902        }
7903        if self.state.gate_ran {
7904            // `review_loop` derives its conclusion from the clean review
7905            // record on every reentry and therefore puts a completed run back
7906            // in `Gating`. A recorded gate is a stronger, terminal fact:
7907            // retain its original command output (or lack of any, for a repo
7908            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
7909            // doc) and restore `Blocked` on a real failure rather than
7910            // pretending the command is still running or running it a second
7911            // time. `gate_ran == false` remains the only shape — unattempted,
7912            // or a resource-blocked retry — that may still need to execute a
7913            // command.
7914            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7915                self.state.status = RunStatus::Blocked;
7916                self.state.save()?;
7917            }
7918            return Ok(());
7919        }
7920        self.state.status = RunStatus::Gating;
7921        let mut outcomes = self.run_gate(&winner).await?;
7922        let mut attempted_head = self.state.gate_head.clone();
7923        loop {
7924            // A resource-blocked outcome means the gate command never actually
7925            // ran - the shared build cache could not be acquired or confirmed
7926            // fresh in time - which is evidence about the machine, not about
7927            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
7928            // Recording it as a red gate would mark a run `Blocked` on nothing
7929            // but contention magi has already logged; leaving `self.state.gate`
7930            // empty and `self.state.gate_ran` false instead keeps the shape
7931            // this function already treats as "still needs to run" (see the
7932            // early-return above), so the next call retries the command
7933            // rather than concluding anything.
7934            if verify_inconclusive(&outcomes) {
7935                self.state.save()?;
7936                return Ok(());
7937            }
7938            if outcomes.iter().all(CommandOutcome::ok) {
7939                break;
7940            }
7941            let fixes_before = self.state.gate_fixes.len();
7942            match self.gate_fix_round(&winner, &outcomes).await? {
7943                GateFix::Retry => {
7944                    outcomes = self.run_gate(&winner).await?;
7945                    attempted_head = self.state.gate_head.clone();
7946                }
7947                GateFix::Stop => {
7948                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
7949                    if self.state.gate_fixes.len() > fixes_before
7950                        && attempted_head.as_deref() != Some(current.as_str())
7951                        && verify_current_head(&mut self.state, &winner, &wait).await?
7952                    {
7953                        // A new SHA owes a gate even when a spent fixer changed only
7954                        // commit metadata. This does not grant another repair round.
7955                        outcomes = self.run_gate(&winner).await?;
7956                        attempted_head = self.state.gate_head.clone();
7957                    } else {
7958                        break;
7959                    }
7960                }
7961                GateFix::Defer => {
7962                    self.state.save()?;
7963                    return Ok(());
7964                }
7965            }
7966        }
7967        let passed = outcomes.iter().all(CommandOutcome::ok);
7968        self.state.gate = outcomes;
7969        self.state.gate_ran = true;
7970        self.state.gate_head = attempted_head;
7971        self.state.save()?;
7972        if !verify_current_head(&mut self.state, &winner, &wait).await? {
7973            return Ok(());
7974        }
7975        if !passed {
7976            self.state.status = RunStatus::Blocked;
7977            let spent = self.state.gate_fixes.len();
7978            self.state.event(
7979                "gate",
7980                if spent == 0 {
7981                    "gate failed; not merging".to_owned()
7982                } else {
7983                    format!("gate failed after {spent} gate-fix round(s); not merging")
7984                },
7985            );
7986        }
7987        self.state.save()?;
7988        Ok(())
7989    }
7990
7991    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7992    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7993    /// after review is clean and never on a candidate awaiting judging.
7994    ///
7995    /// Never fails the run: a non-zero exit or timeout is a warning and a
7996    /// recorded outcome, and the gate remains the single arbiter. Nothing
7997    /// configured means nothing happens - no event, no commit. `commit_all`
7998    /// commits any leftover change under the neutral identity and returns
7999    /// `false` when the tree is clean, so no empty commit is ever made.
8000    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8001        let commands = state.config.verify.pre_gate.clone();
8002        if commands.is_empty() {
8003            return;
8004        }
8005        let shell = state.config.shell();
8006        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8007        let (outcomes, _) = run_commands(
8008            state,
8009            "pre_gate",
8010            "pre_gate",
8011            0,
8012            &shell,
8013            &commands,
8014            &winner.worktree,
8015            timeout,
8016        )
8017        .await;
8018        for o in &outcomes {
8019            if !o.ok() {
8020                tracing::warn!(
8021                    "pre_gate `{}` failed ({:?}); the gate decides",
8022                    o.command,
8023                    o.code
8024                );
8025            }
8026            state.event(
8027                "pre_gate",
8028                format!(
8029                    "`{}` -> {}",
8030                    o.command,
8031                    if o.ok() {
8032                        "pass".to_owned()
8033                    } else {
8034                        format!(
8035                            "FAIL ({:?})\n{}",
8036                            o.code,
8037                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8038                        )
8039                    }
8040                ),
8041            );
8042        }
8043        state.pre_gate = outcomes;
8044        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8045            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8046                Ok(head) => {
8047                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8048                    state.pre_gate_commit = Some(head);
8049                }
8050                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8051            },
8052            Ok(false) => {}
8053            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8054        }
8055        if let Err(e) = state.save() {
8056            tracing::warn!("could not persist the pre_gate record: {e:#}");
8057        }
8058    }
8059
8060    /// Run `verify.gate` once against the winner's current tree, logging one
8061    /// event per command. Empty when nothing is configured.
8062    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8063        let wait = self.repository_wait();
8064        run_landing_gate(&mut self.state, winner, &wait).await
8065    }
8066
8067    /// One bounded fix round for a failing gate.
8068    ///
8069    /// The fixer is told the failure came from the gate itself, not from a
8070    /// reviewer, and is shown the failed commands, their exit codes and a tail
8071    /// of their output - whatever `[verify].gate` holds, nothing here knows
8072    /// what those commands run. Only a normal non-zero exit that printed
8073    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8074    /// command or a full disk says nothing about the code, and a fixer sent
8075    /// after it can only appease the machine. The round is judged by what git
8076    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8077    /// again before the gate does, so a fix cannot trade a green gate for a
8078    /// red e2e unnoticed.
8079    async fn gate_fix_round(
8080        &mut self,
8081        winner: &Candidate,
8082        outcomes: &[CommandOutcome],
8083    ) -> Result<GateFix> {
8084        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8085            g.phase == crate::repair_budget::Phase::Fixing
8086                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8087        }) {
8088            self.state.gate_fixes.len().saturating_add(1)
8089        } else {
8090            self.state.config.graph.gate_fix_rounds
8091        };
8092        let spent = self.state.gate_fixes.len();
8093        if spent >= cap {
8094            if cap > 0 {
8095                self.state.event(
8096                    "gate",
8097                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8098                );
8099            }
8100            return Ok(GateFix::Stop);
8101        }
8102        if !gate_fixable(outcomes) {
8103            self.state.event(
8104                "gate",
8105                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8106                 command or similar); not spending a fix round on it",
8107            );
8108            return Ok(GateFix::Stop);
8109        }
8110        let min_free = self.state.config.disk.min_free_bytes;
8111        if min_free > 0 {
8112            match crate::disk::free_bytes(&winner.worktree) {
8113                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8114                Ok(free) => {
8115                    self.state.event(
8116                        "gate",
8117                        format!(
8118                            "only {free} bytes free ({min_free} required by `[disk] \
8119                             min_free_bytes`); not spending a fix round on a failure the disk \
8120                             may explain"
8121                        ),
8122                    );
8123                    return Ok(GateFix::Stop);
8124                }
8125                Err(e) => {
8126                    self.state.event(
8127                        "gate",
8128                        format!("free disk space could not be measured ({e:#}); no fix round"),
8129                    );
8130                    return Ok(GateFix::Stop);
8131                }
8132            }
8133        }
8134
8135        let attempt = spent + 1;
8136        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8137        let base = self.landing_base();
8138        let fix_prompt = prompt::gate_fix_in(
8139            &self.state.instruction,
8140            &failed,
8141            attempt,
8142            cap,
8143            &self.state.config.graph.language,
8144            &self.state.config.graph.github_language,
8145        );
8146        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8147        let sessions = self.state.config.graph.sessions;
8148        let artifacts = self.merge_repair_artifacts();
8149        self.state.event(
8150            "gate",
8151            format!("gate failed; gate-fix round {attempt} of {cap}"),
8152        );
8153        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8154        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8155        // Persist before invoking the fixer: it may commit and then be
8156        // interrupted before its result (or the following e2e) is saved.
8157        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8158            head: before.clone(),
8159            passed: false,
8160            outcomes: Vec::new(),
8161        });
8162        self.state.save()?;
8163        let (_, seat, out) = self
8164            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8165                prompt: fix_prompt.clone(),
8166                spec,
8167                seat,
8168                cwd: winner.worktree.clone(),
8169                timeout,
8170                allow_write: true,
8171                sessions,
8172                artifacts: artifacts.clone(),
8173                stem: format!("gate-fix-{attempt}"),
8174                handover: None,
8175            })
8176            .await;
8177        // The fixer was never asked: no round is spent (nothing is recorded)
8178        // and a later reentry asks the same agent again.
8179        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8180            return Ok(GateFix::Defer);
8181        }
8182        let mut record = GateFixRecord {
8183            agent: seat.agent.clone(),
8184            failed,
8185            notes: String::new(),
8186            committed: false,
8187            error: None,
8188        };
8189        match out {
8190            AgentOutcome::Ok(o) => {
8191                // A missing report is not a failed fix: the round is judged
8192                // by the tree below, and the report only carries prose.
8193                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8194                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8195                }
8196            }
8197            AgentOutcome::Dropped(_) => {
8198                record.error = Some("the CLI dropped the stream".to_owned());
8199            }
8200            AgentOutcome::Quota(o) => {
8201                self.state.quota.push(QuotaLoss::from_report(
8202                    seat.key.clone(),
8203                    "gate-fix",
8204                    o.quota.as_ref(),
8205                ));
8206                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8207            }
8208            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8209        }
8210        self.state.seats.insert(seat.key.clone(), seat);
8211        let coordination = self.coordinate(&winner.worktree).await?;
8212        if let Ok(r) = git::rescue_commit(
8213            &winner.worktree,
8214            &format!("magi: gate fix {attempt} (uncommitted work)"),
8215        )
8216        .await
8217        {
8218            self.state.note_withheld("gate-fix", &r.withheld);
8219        }
8220        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8221        record.committed = after != before;
8222        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8223        let note = record.error.clone();
8224        self.state.gate_fixes.push(record);
8225        self.state.save()?;
8226        if !changed {
8227            self.state.event(
8228                "gate",
8229                match note {
8230                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8231                    None => format!("gate-fix round {attempt}: the tree did not change"),
8232                },
8233            );
8234            return Ok(GateFix::Stop);
8235        }
8236        self.state.event(
8237            "gate",
8238            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8239        );
8240
8241        drop(coordination);
8242        let wait = self.repository_wait();
8243        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8244    }
8245
8246    // ---------------------------------------------------------------- merge
8247
8248    /// One read-only rewrite by the summary's author; a text the gate still
8249    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8250    /// applies on their word or on silence. `None` means the run is parked on
8251    /// that question and nothing may be pushed or opened yet.
8252    async fn guarded_pr_message(
8253        &mut self,
8254        winner: &Candidate,
8255        facts: Option<&BranchFacts>,
8256        posting: bool,
8257    ) -> Result<Option<PrMessage>> {
8258        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8259        let head = match final_head.as_deref() {
8260            Some(h) => crate::pr_outcome::FinalHead::At(h),
8261            None => crate::pr_outcome::FinalHead::Unknown,
8262        };
8263        // Only a pull request that is about to be posted pays for a summary.
8264        let summarized = if posting {
8265            Some(self.summarize_change(winner, final_head.as_deref()).await)
8266        } else {
8267            None
8268        };
8269        // A busy shared resource is not a permanent cause: publishing an
8270        // "unavailable" description would stick (a later head move never
8271        // edits the body). Stop before anything is pushed or opened; the
8272        // run is `Blocked` and a resume asks the summary again.
8273        if self.state.resource_block.is_some() {
8274            self.state.event(
8275                "pr-summary",
8276                "a shared resource was busy; nothing was published, resume to summarize again",
8277            );
8278            return Ok(None);
8279        }
8280        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8281            anyhow::bail!(
8282                "the final commit moved during public description generation; retry before publishing"
8283            );
8284        }
8285        let note = match (&summarized, &self.state.change_summary) {
8286            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8287            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8288            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8289            (None, _) => ChangeNote::NotAsked,
8290        };
8291        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8292        if !posting {
8293            let identity = crate::scrub::Identity::current();
8294            return Ok(Some(PrMessage {
8295                title: crate::scrub::scrub(&pr.title, &identity),
8296                body: crate::scrub::scrub(&pr.body, &identity),
8297            }));
8298        }
8299        // This very text was already rejected and asked about: no second
8300        // judge call, rewrite or question, just carry the record on.
8301        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8302        if let Some(known) = self
8303            .state
8304            .github_text
8305            .clone()
8306            .filter(|g| g.fingerprint == raw_fp)
8307        {
8308            return self.settle_withheld(winner, &pr, known).await;
8309        }
8310        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8311        let mut violations =
8312            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8313        let identity = crate::scrub::Identity::current();
8314        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8315            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8316            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8317        {
8318            violations.push(crate::github_text::Violation::SensitiveData);
8319        }
8320        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8321        // only a language violation starts one; sensitive data alone goes
8322        // straight to the owner, with the redacted text as the fallback.
8323        let has_language = violations
8324            .iter()
8325            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8326        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8327            self.state.event(
8328                "github-text",
8329                if has_language {
8330                    format!("description rejected: {violations:?}; requesting one rewrite")
8331                } else {
8332                    format!("description rejected: {violations:?}; asking the owner")
8333                },
8334            );
8335            let mut rewritten = false;
8336            let mut rejected_rewrite = None;
8337            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8338            {
8339                let output = text;
8340                if let Some(title) = summary_title(&output) {
8341                    // Judge the redacted candidate that will actually be posted.
8342                    // Sensitive data alone must not discard a useful rewrite.
8343                    let raw_body = summary_without_title(&output);
8344                    let clean_title = crate::scrub::scrub(&title, &identity);
8345                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8346                    let redacted = clean_title != title || body != raw_body;
8347                    let title = clean_title;
8348                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8349                    if !complete {
8350                        self.state.event(
8351                            "github-text",
8352                            "description rewrite omitted required findings",
8353                        );
8354                    }
8355                    if !body.trim().is_empty() {
8356                        body.push_str("\n\n");
8357                        body.push_str(&public_pr_evidence(&self.state, head));
8358                    }
8359                    let marker = format!("magi:run/{}", self.state.id);
8360                    if !body.trim().is_empty() && !body.contains(&marker) {
8361                        body.push_str(&format!("\n\n{marker}"));
8362                    }
8363                    let rewrite = PrMessage {
8364                        title: title.clone(),
8365                        body: body.clone(),
8366                    };
8367                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8368                    let mut failed = crate::github_text::check_for(
8369                        &self.state.config.graph,
8370                        &title,
8371                        &body,
8372                        redecision,
8373                    );
8374                    if (!crate::github_text::shareable(&title)
8375                        || !crate::github_text::shareable(&body))
8376                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8377                    {
8378                        failed.push(crate::github_text::Violation::SensitiveData);
8379                    }
8380                    if complete
8381                        && !title.trim().is_empty()
8382                        && !body.trim().is_empty()
8383                        && !failed.is_empty()
8384                    {
8385                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8386                            title: title.clone(),
8387                            body: body.clone(),
8388                            failed: failed
8389                                .iter()
8390                                .map(|v| crate::github_text::category(*v).to_owned())
8391                                .collect(),
8392                        });
8393                    }
8394                    if complete
8395                        && !title.trim().is_empty()
8396                        && !raw_body.trim().is_empty()
8397                        && failed.is_empty()
8398                    {
8399                        decision = redecision;
8400                        pr = PrMessage { title, body };
8401                        rewritten = true;
8402                        self.state
8403                            .event("github-text", "description rewrite passed");
8404                        if redacted {
8405                            self.state.event(
8406                                "github-text",
8407                                "sensitive data removed from description rewrite",
8408                            );
8409                        }
8410                    }
8411                }
8412            }
8413            if !rewritten {
8414                self.state.event(
8415                    "github-text",
8416                    "description rewrite unavailable or rejected; asking the owner",
8417                );
8418                if let Some(w) =
8419                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8420                {
8421                    // A different text than the one asked about before: that
8422                    // question no longer describes anything.
8423                    self.retire_github_text_question("the withheld text changed");
8424                    let record = crate::run::GithubTextAsk {
8425                        rewrite: rejected_rewrite,
8426                        fingerprint: raw_fp,
8427                        title: w.title,
8428                        body: w.body,
8429                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8430                        question: None,
8431                        asks: 0,
8432                        resolved: false,
8433                        chosen_title: None,
8434                        choice: None,
8435                    };
8436                    return self.settle_withheld(winner, &pr, record).await;
8437                }
8438            }
8439        }
8440        let (title, body) =
8441            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8442        Ok(Some(PrMessage { title, body }))
8443    }
8444
8445    /// Describe the whole change from its final diff in either run mode.
8446    ///
8447    /// The summary is tied to the final commit and to the merge base of the
8448    /// resolved remote base; a record for any other pair is dropped and the
8449    /// agent asked again, so a description never outlives the diff it was
8450    /// written from. The seat is read-only and the prompt tells it to read the
8451    /// diff and the code around it. `[roles] synthesizer` selects the agents
8452    /// (unset: the default pick), each asked at most once, advancing on an
8453    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8454    /// carries the reason it is unavailable; nothing is invented then.
8455    async fn summarize_change(
8456        &mut self,
8457        winner: &Candidate,
8458        head: Option<&str>,
8459    ) -> std::result::Result<(), String> {
8460        let Some(head) = head else {
8461            return Err("the final commit could not be read".to_owned());
8462        };
8463        let remote = self.state.config.merge.remote.clone();
8464        let tracking = format!("{remote}/{}", self.state.base_branch);
8465        if !git::rev_exists(&winner.worktree, &tracking).await {
8466            return Err(format!("the remote base `{tracking}` could not be read"));
8467        }
8468        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8469            Ok(b) if !b.is_empty() => b,
8470            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8471        };
8472        let evidence = crate::pr_outcome::summary_evidence(
8473            &self.state,
8474            crate::pr_outcome::FinalHead::At(head),
8475        );
8476        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8477        if self
8478            .state
8479            .change_summary
8480            .as_ref()
8481            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8482        {
8483            return Ok(());
8484        }
8485        if self.state.change_summary.take().is_some() {
8486            self.state.event(
8487                "pr-summary",
8488                "the final diff or evidence changed; the earlier change summary was discarded",
8489            );
8490        }
8491        // Save the source snapshot even if no generation agent is available.
8492        if let Err(e) = write_artifact(
8493            &self.state,
8494            &format!("public-description-{fingerprint}.md"),
8495            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8496        ) {
8497            tracing::warn!("could not save public description evidence: {e:#}");
8498        }
8499        let chain = match agent::pick_chain(
8500            &self.state.config.agents,
8501            self.state.config.roles.synthesizer.as_ref(),
8502            &agent::installed,
8503            "synthesizer",
8504        ) {
8505            Ok(chain) => chain,
8506            Err(e) => {
8507                tracing::warn!("no agent for the change summary: {e:#}");
8508                self.state.event("pr-summary", "no summary agent available");
8509                return Err("no agent was available to summarize it".to_owned());
8510            }
8511        };
8512        let subjects = git::subjects(&winner.worktree, &base, head)
8513            .await
8514            .unwrap_or_default();
8515        let stat = git::diff_stat(&winner.worktree, &base, head)
8516            .await
8517            .unwrap_or_default();
8518        let stat: String = stat.chars().take(6_000).collect();
8519        let mut prompt = crate::prompt::change_summary_in(
8520            &base,
8521            head,
8522            &subjects,
8523            stat.trim(),
8524            &self.state.config.graph.github_language,
8525        );
8526        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8527        let sessions = self.state.config.graph.sessions;
8528        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8529        let prompts = self.state.config.prompts.clone();
8530        let cache = self.state.config.cache_dir();
8531        let run = self.state.id.clone();
8532        for (i, spec) in chain.into_iter().enumerate() {
8533            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8534            let github_spec = match agent::for_github_text(
8535                &spec,
8536                &self.state.config.graph.github_language,
8537                &winner.worktree,
8538            ) {
8539                Ok(spec) => spec,
8540                Err(e) => {
8541                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8542                    continue;
8543                }
8544            };
8545            let job = SeatJob {
8546                spec: github_spec,
8547                seat,
8548                cwd: winner.worktree.clone(),
8549                prompt: prompt.clone(),
8550                timeout,
8551                allow_write: false,
8552                sessions,
8553                artifacts: agent::artifacts_dir(&self.state.dir()),
8554                stem: if i == 0 {
8555                    "change-summary".to_owned()
8556                } else {
8557                    format!("change-summary-{}", spec.id)
8558                },
8559                handover: None,
8560            };
8561            let ctx = WaveCtx {
8562                run: &run,
8563                node: "pr-summary",
8564                prompts: &prompts,
8565                cache: cache.as_deref(),
8566                round: None,
8567                carry_seats: false,
8568                pauses: Some((&self.pause, &self.interrupt)),
8569            };
8570            let (seat, outcome) =
8571                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8572            self.state.seats.insert(seat.key.clone(), seat);
8573            let text = match outcome {
8574                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8575                AgentOutcome::ResourceBlocked(why) => {
8576                    // No agent was asked: the run stops resumably before
8577                    // anything is published (see `guarded_pr_message`).
8578                    if self.state.resource_block.is_none() {
8579                        self.state.resource_block = Some(crate::run::ResourceBlock {
8580                            cause: crate::run::ResourceBlockCause::Other,
8581                            node: "pr-summary".to_owned(),
8582                            kind: "shared resource".to_owned(),
8583                            detail: why,
8584                            at: Timestamp::now(),
8585                        });
8586                    }
8587                    return Err("a shared resource was busy".to_owned());
8588                }
8589                _ => continue,
8590            };
8591            let blind = &self.state.config.blind;
8592            let Some(title) = summary_title(&text) else {
8593                continue;
8594            };
8595            let body: String = summary_without_title(&text)
8596                .lines()
8597                .filter(|l| !l.contains("magi:run/"))
8598                .collect::<Vec<_>>()
8599                .join("\n");
8600            let body = blind::sanitize_prose(body.trim(), blind);
8601            let title = blind::sanitize_prose(&title, blind);
8602            if body.trim().is_empty() || title.trim().is_empty() {
8603                continue;
8604            }
8605            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8606                continue;
8607            }
8608            // The branch must still be the one the agent was shown.
8609            if git::rev_parse(&winner.worktree, "HEAD")
8610                .await
8611                .ok()
8612                .as_deref()
8613                != Some(head)
8614            {
8615                return Err("the branch moved while the summary was written".to_owned());
8616            }
8617            if git::merge_base(&winner.worktree, &tracking, head)
8618                .await
8619                .ok()
8620                .as_deref()
8621                != Some(base.as_str())
8622                || crate::pr_outcome::summary_evidence(
8623                    &self.state,
8624                    crate::pr_outcome::FinalHead::At(head),
8625                ) != evidence
8626            {
8627                return Err("the summary evidence changed while it was written".to_owned());
8628            }
8629            self.state.change_summary = Some(crate::run::ChangeSummary {
8630                head: head.to_owned(),
8631                base,
8632                evidence: fingerprint,
8633                title,
8634                body,
8635            });
8636            self.state.event(
8637                "pr-summary",
8638                format!("change summary written by {} for the final diff", spec.id),
8639            );
8640            if let Err(e) = self.state.save() {
8641                tracing::warn!("could not save the change summary: {e:#}");
8642            }
8643            return Ok(());
8644        }
8645        self.state
8646            .event("pr-summary", "no agent produced a change summary");
8647        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8648    }
8649
8650    /// One read-only rewrite request for a withheld description. The
8651    /// implementer's own conversation is tried first when it exists and can be
8652    /// resumed; then the `[roles] language_judge` chain (or the default pick)
8653    /// writes it in a separate conversation, skipping the implementer's agent.
8654    /// The prompt carries the whole text, so no agent needs any memory. Each
8655    /// agent is asked at most once and the chain advances on an error, quota,
8656    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
8657    /// that has run out of quota must not end the rewrite. Nothing is recorded
8658    /// as a handover.
8659    async fn rewrite_pr_text(
8660        &mut self,
8661        winner: &Candidate,
8662        pr: &PrMessage,
8663        violations: &[crate::github_text::Violation],
8664    ) -> Option<String> {
8665        let prompt = format!(
8666            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8667            pr.title, pr.body
8668        );
8669        let sessions = self.state.config.graph.sessions;
8670        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8671        // (spec, the resumed implementer's seat). A fallback seat is created
8672        // only when its turn comes, so an unused fallback leaves no record.
8673        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8674        let key = format!("impl-{}", winner.label);
8675        if let Some(spec) = self
8676            .state
8677            .config
8678            .agents
8679            .iter()
8680            .find(|a| a.id == winner.agent)
8681            .cloned()
8682            && let Some(existing) = self.state.seats.get(&key)
8683            && existing.agent == spec.id
8684            && agent::has_session(spec.kind, existing, sessions)
8685        {
8686            attempts.push((spec, Some(existing.clone())));
8687        }
8688        match agent::pick_chain(
8689            &self.state.config.agents,
8690            self.state.config.roles.language_judge.as_ref(),
8691            &agent::installed,
8692            "language_judge",
8693        ) {
8694            Ok(chain) => {
8695                for spec in chain {
8696                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8697                        continue;
8698                    }
8699                    attempts.push((spec, None));
8700                }
8701            }
8702            Err(e) if attempts.is_empty() => {
8703                tracing::warn!("no agent for the description rewrite: {e:#}");
8704                self.state
8705                    .event("github-text", "no rewrite agent available");
8706                return None;
8707            }
8708            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8709        }
8710        let prompts = self.state.config.prompts.clone();
8711        let cache = self.state.config.cache_dir();
8712        let run = self.state.id.clone();
8713        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8714            let resumed = resumed_seat.is_some();
8715            let seat = resumed_seat.unwrap_or_else(|| {
8716                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8717            });
8718            let github_spec = match agent::for_github_text(
8719                &spec,
8720                &self.state.config.graph.github_language,
8721                &winner.worktree,
8722            ) {
8723                Ok(spec) => spec,
8724                Err(e) => {
8725                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8726                    continue;
8727                }
8728            };
8729            let job = SeatJob {
8730                spec: github_spec,
8731                seat,
8732                cwd: winner.worktree.clone(),
8733                prompt: prompt.clone(),
8734                timeout: retry_budget(timeout_full, resumed),
8735                allow_write: false,
8736                sessions,
8737                artifacts: agent::artifacts_dir(&self.state.dir()),
8738                stem: if i == 0 {
8739                    "github-text-rewrite".to_owned()
8740                } else {
8741                    format!("github-text-rewrite-{}", spec.id)
8742                },
8743                handover: None,
8744            };
8745            let ctx = WaveCtx {
8746                run: &run,
8747                node: "github-text",
8748                prompts: &prompts,
8749                cache: cache.as_deref(),
8750                round: None,
8751                carry_seats: false,
8752                pauses: Some((&self.pause, &self.interrupt)),
8753            };
8754            let (seat, outcome) =
8755                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8756            self.state.seats.insert(seat.key.clone(), seat);
8757            match outcome {
8758                AgentOutcome::Ok(o)
8759                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8760                {
8761                    return Some(o.text);
8762                }
8763                AgentOutcome::ResourceBlocked(_) => return None,
8764                _ => {}
8765            }
8766        }
8767        None
8768    }
8769
8770    /// Abandon the standing posting-gate question, if any.
8771    fn retire_github_text_question(&mut self, why: &str) {
8772        let Some(id) = self
8773            .state
8774            .github_text
8775            .as_ref()
8776            .and_then(|g| g.question.clone())
8777        else {
8778            return;
8779        };
8780        let _ = ask::Questions::open().update(&id, |q| {
8781            q.abandon(why);
8782            Ok(())
8783        });
8784    }
8785
8786    /// Carry a withheld text to its end: file the question, park while it is
8787    /// open, and on an answer, silence or timeout pick the text to post. The
8788    /// decision is saved before it is used, so it applies once.
8789    async fn settle_withheld(
8790        &mut self,
8791        winner: &Candidate,
8792        pr: &PrMessage,
8793        mut g: crate::run::GithubTextAsk,
8794    ) -> Result<Option<PrMessage>> {
8795        use crate::github_text as gt;
8796        let store = ask::Questions::open();
8797        let timeout = self.state.config.graph.answer_timeout;
8798        if !g.resolved {
8799            let mut standing = g
8800                .question
8801                .as_ref()
8802                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8803            if let Some(q) = standing.as_ref()
8804                && q.status.open()
8805                && gt::expired(q, timeout)
8806            {
8807                let why = format!("no answer within {}s of asking", timeout.max(1));
8808                standing = store
8809                    .update(&q.id, |q| {
8810                        q.abandon(&why);
8811                        Ok(())
8812                    })
8813                    .ok()
8814                    .map(|(q, ())| q);
8815            }
8816            let mut ask_again: Option<Option<Vec<String>>> = None;
8817            match standing {
8818                // Recorded but never written (a stop between the two saves),
8819                // or removed: ask again without spending the retry.
8820                None if g.asks <= 1 => {
8821                    g.asks = 0;
8822                    ask_again = Some(None);
8823                }
8824                None => {
8825                    g.resolved = true;
8826                    g.chosen_title = None;
8827                    g.choice = Some(GithubTextChoice::Fallback);
8828                }
8829                Some(q) if q.status.open() => {
8830                    self.park_on_github_text(&g)?;
8831                    return Ok(None);
8832                }
8833                Some(q) => match gt::read_reply(&q) {
8834                    gt::Reply::Fallback => {
8835                        g.resolved = true;
8836                        g.chosen_title = None;
8837                        g.choice = Some(GithubTextChoice::Fallback);
8838                    }
8839                    gt::Reply::Original => {
8840                        g.resolved = true;
8841                        g.chosen_title = None;
8842                        g.choice = Some(GithubTextChoice::Original);
8843                    }
8844                    gt::Reply::Rewrite => {
8845                        g.resolved = true;
8846                        g.chosen_title = None;
8847                        g.choice = Some(if g.rewrite.is_some() {
8848                            GithubTextChoice::Rewrite
8849                        } else {
8850                            GithubTextChoice::Fallback
8851                        });
8852                    }
8853                    gt::Reply::Title(text) => {
8854                        let first = text
8855                            .lines()
8856                            .map(str::trim)
8857                            .find(|l| !l.is_empty())
8858                            .unwrap_or("");
8859                        let decision = if first.is_empty() {
8860                            None
8861                        } else {
8862                            let candidate = PrMessage {
8863                                title: first.to_owned(),
8864                                body: String::new(),
8865                            };
8866                            self.judge_pr_language(&winner.worktree, &candidate).await
8867                        };
8868                        match gt::vet_title(
8869                            &text,
8870                            decision,
8871                            self.state.config.graph.github_english(),
8872                        ) {
8873                            Ok(title) => {
8874                                g.resolved = true;
8875                                g.chosen_title = Some(title);
8876                                g.choice = Some(GithubTextChoice::MyTitle);
8877                            }
8878                            Err(cats) if g.asks < 2 => {
8879                                ask_again = Some(Some(
8880                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8881                                ))
8882                            }
8883                            Err(_) => {
8884                                g.resolved = true;
8885                                g.chosen_title = None;
8886                                g.choice = Some(GithubTextChoice::Fallback);
8887                            }
8888                        }
8889                    }
8890                },
8891            }
8892            if let Some(rejected) = ask_again {
8893                // A retry (after a rejected replacement title) is still about
8894                // the original text: the same ORIGINAL / FALLBACK comparison.
8895                let w = gt::Withheld {
8896                    title: g.title,
8897                    body: g.body,
8898                    categories: g.categories.clone(),
8899                };
8900                // Named by the original text's fingerprint alone, so a resume
8901                // rewrites the same file and the detail stays identical.
8902                let artifact = match crate::run::write_artifact(
8903                    &self.state,
8904                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
8905                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8906                ) {
8907                    Ok(p) => Some(p),
8908                    Err(e) => {
8909                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
8910                        None
8911                    }
8912                };
8913                let (detail, comparison) = gt::question_content(
8914                    &w,
8915                    &pr.title,
8916                    &pr.body,
8917                    &self.state.id,
8918                    rejected.as_deref(),
8919                    artifact.as_deref(),
8920                    g.rewrite.as_ref(),
8921                );
8922                let mut q = ask::Question::new(
8923                    self.state.id.clone(),
8924                    gt::ASK_NODE.to_owned(),
8925                    gt::ASK_SEAT.to_owned(),
8926                    gt::question_summary(&self.state.config.graph.language, &w),
8927                    detail,
8928                    gt::question_choices(&w, g.rewrite.as_ref()),
8929                );
8930                q.posting_comparison = Some(comparison);
8931                q.answer_timeout = timeout;
8932                // Recorded before it is written: a stop in between leaves a
8933                // record naming a question the resume then files again.
8934                g.question = Some(q.id.clone());
8935                g.asks += 1;
8936                self.state.github_text = Some(g.clone());
8937                self.state.event(
8938                    "github-text",
8939                    format!("asking the owner about the withheld text ({})", q.short()),
8940                );
8941                self.state.save()?;
8942                store
8943                    .put_executable(&mut q)
8944                    .context("file the posting-gate question")?;
8945                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8946                    tracing::warn!(
8947                        "could not notify about posting-gate question {}: {e:#}",
8948                        q.short()
8949                    );
8950                }
8951                self.park_on_github_text(&g)?;
8952                return Ok(None);
8953            }
8954        }
8955        let resolved = g.chosen_title.clone();
8956        let original = g.choice == Some(GithubTextChoice::Original);
8957        // Field names only: the event log never records the text.
8958        self.state.event(
8959            "github-text",
8960            if original {
8961                format!(
8962                    "posting the original text for: {}",
8963                    match (g.title, g.body) {
8964                        (true, true) => "title, description",
8965                        (true, false) => "title",
8966                        _ => "description",
8967                    }
8968                )
8969            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8970                "posting the saved rewrite title and description".to_owned()
8971            } else if resolved.is_some() {
8972                "posting the owner's replacement title".to_owned()
8973            } else {
8974                "posting the neutral text for the withheld fields".to_owned()
8975            },
8976        );
8977        self.state.github_text = Some(g.clone());
8978        self.state.save()?;
8979        if g.choice == Some(GithubTextChoice::Rewrite)
8980            && let Some(r) = g.rewrite.as_ref()
8981        {
8982            return Ok(Some(PrMessage {
8983                title: r.title.clone(),
8984                body: r.body.clone(),
8985            }));
8986        }
8987        let w = gt::Withheld {
8988            title: g.title,
8989            body: g.body,
8990            categories: g.categories.clone(),
8991        };
8992        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8993        if let Some(t) = resolved {
8994            title = t;
8995        }
8996        // Every withheld field is now neutral or vetted English.
8997        let english = gt::LanguageDecision {
8998            title_english: true,
8999            body_english: true,
9000            body_complete: true,
9001        };
9002        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9003        if original {
9004            // The owner's explicit choice overrides the gate for the fields it
9005            // was made for; the others keep what the gate produced.
9006            if g.title {
9007                title = pr.title.clone();
9008            }
9009            if g.body {
9010                body = pr.body.clone();
9011            }
9012        }
9013        Ok(Some(PrMessage { title, body }))
9014    }
9015
9016    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9017        self.state.github_text = Some(g.clone());
9018        self.state.end_pass_parked(ParkCause::Question);
9019        self.state.parked = true;
9020        self.state.event(
9021            "github-text",
9022            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9023        );
9024        self.state.save()
9025    }
9026
9027    /// Ask `[roles] language_judge` about this text and record which source
9028    /// decided; the text itself is never recorded.
9029    async fn judge_pr_language(
9030        &mut self,
9031        cwd: &Path,
9032        pr: &PrMessage,
9033    ) -> Option<crate::github_text::LanguageDecision> {
9034        self.state.config.roles.language_judge.as_ref()?;
9035        if !self.state.config.graph.language_gate() {
9036            return None;
9037        }
9038        let decision =
9039            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9040        self.state.event(
9041            "github-text",
9042            if decision.is_some() {
9043                "language decided by the language judge"
9044            } else {
9045                "language judge unavailable; using built-in heuristics"
9046            },
9047        );
9048        decision
9049    }
9050
9051    async fn merge(&mut self) -> Result<()> {
9052        if self.resource_halt()? || self.hold_base_sync().await? {
9053            return Ok(());
9054        }
9055        // Same reasoning as `gate`: ask the review and gate records directly
9056        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9057        // skip can rewrite on reentry to something that no longer says
9058        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9059        // so a hand-off (open findings, green verification) reaches merge
9060        // exactly like a genuinely clean round does.
9061        //
9062        // A run resumed mid-`land` never reaches here at all: `execute`
9063        // recognises `RunStatus::Landing` before it even calls `prep`, and
9064        // routes straight to `run_land` instead. That has to happen a level
9065        // up from this function, not with a check in here, because
9066        // `review_loop`'s own status recomputation (see its doc) runs
9067        // *before* `merge` on every reentry and would otherwise overwrite
9068        // the `Landing` marker with `Gating` before this node ever saw it.
9069        if self
9070            .state
9071            .base_sync
9072            .as_ref()
9073            .is_some_and(|s| s.conflict.is_some())
9074            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9075                != Some(RunStatus::Gating)
9076            // `gate_ran == false` is not "passed" - `gate` leaves it false
9077            // both before it has ever run and when its last attempt was
9078            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9079            // permission to merge on nothing but the review record. Only a
9080            // gate that actually ran may proceed; an intentional local
9081            // review can also finish with verification explicitly not configured.
9082            // `RunState::gate_status` computes that distinction.
9083            || !self.state.gate_allows_progress()
9084        {
9085            return Ok(());
9086        }
9087        // This node's own record, not `status`: `status == Ready` is not
9088        // unique to the harmless `MergeMode::None` path this line was
9089        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9090        // run's PR was closed without merging — and on that run `mode` is
9091        // still `Pr`, so a reentry that fell through here would push and
9092        // open a second pull request. `self.state.merge` is set exactly once
9093        // this node (or `land`) has already produced a verdict, under every
9094        // mode, which is what "already done" actually means here.
9095        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9096            return Ok(());
9097        }
9098        let Some(winner) = self.state.winner().cloned() else {
9099            return Ok(());
9100        };
9101        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9102        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9103            self.state.status = RunStatus::Blocked;
9104            self.state.event(
9105                "gate",
9106                "winner changed after gate; fresh verification required",
9107            );
9108            self.state.save()?;
9109            return Ok(());
9110        }
9111        let wait = self.repository_wait();
9112        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9113            return Ok(());
9114        }
9115        let repo = self.state.repo.clone();
9116        let base = self.state.base_branch.clone();
9117        let mode = self.state.config.merge.mode;
9118        let style = self.state.config.merge.style;
9119        let facts = if is_review_run(&self.state) {
9120            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9121            let start = review_base(
9122                &repo,
9123                &self.state.config.merge.remote,
9124                &base,
9125                &self.state.base_commit,
9126                &winner.branch,
9127            )
9128            .await;
9129            branch_facts(&repo, &start, &winner.branch).await
9130        } else {
9131            None
9132        };
9133        // `None` when the base could not be freshly read: then an adopted
9134        // pull request's title is left alone.
9135        let leaked = if is_review_run(&self.state) {
9136            leaked_subjects(&self.state, &winner.branch).await
9137        } else {
9138            Some(Vec::new())
9139        };
9140        // `None`: parked on the owner's word about a withheld title or body.
9141        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9142        // the resume arrives here again.
9143        let Some(pr) = self
9144            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9145            .await?
9146        else {
9147            return Ok(());
9148        };
9149        let message = pr.commit_message();
9150
9151        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9152        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9153            return Ok(());
9154        }
9155        let outcome = match mode {
9156            MergeMode::None => MergeOutcome {
9157                mode,
9158                ok: true,
9159                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9160                empty: false,
9161            },
9162            MergeMode::Pr | MergeMode::Local
9163                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9164            {
9165                MergeOutcome {
9166                    mode,
9167                    ok: false,
9168                    detail: empty_candidate_detail(&self.state, &base),
9169                    empty: true,
9170                }
9171            }
9172            MergeMode::Local => {
9173                let on = git::current_branch(&repo).await?;
9174                if on.as_deref() != Some(base.as_str()) {
9175                    MergeOutcome {
9176                        mode,
9177                        ok: false,
9178                        detail: format!(
9179                            "{} has {} checked out, not the base branch {base}",
9180                            repo.display(),
9181                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9182                        ),
9183                        empty: false,
9184                    }
9185                } else if !git::is_clean(&repo).await? {
9186                    MergeOutcome {
9187                        mode,
9188                        ok: false,
9189                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9190                        empty: false,
9191                    }
9192                } else {
9193                    let out = match style {
9194                        MergeStyle::Merge => {
9195                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9196                        }
9197                        MergeStyle::Squash => {
9198                            git::merge_squash(&repo, &winner.branch, &message).await?
9199                        }
9200                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9201                    };
9202                    MergeOutcome {
9203                        mode,
9204                        ok: out.ok(),
9205                        detail: if out.ok() { out.stdout } else { out.stderr },
9206                        empty: false,
9207                    }
9208                }
9209            }
9210            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9211                self.state.pr_publication_pending = true;
9212                self.state.save()?;
9213                let out =
9214                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9215                self.state.pr_publication_pending = out.is_err();
9216                MergeOutcome {
9217                    mode,
9218                    ok: out.is_ok(),
9219                    detail: match out {
9220                        Ok(url) => url,
9221                        Err(e) => format!("{e:#}"),
9222                    },
9223                    empty: false,
9224                }
9225            }
9226            MergeMode::Pr => {
9227                let remote = self.state.config.merge.remote.clone();
9228                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9229                let pushed = if self.state.foreign_checkout.is_some() {
9230                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9231                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9232                } else {
9233                    git::push(&winner.worktree, &remote, &winner.branch).await?
9234                };
9235                if !pushed.ok() {
9236                    MergeOutcome {
9237                        mode,
9238                        ok: false,
9239                        detail: pushed.stderr,
9240                        empty: false,
9241                    }
9242                } else {
9243                    if self.state.foreign_checkout.is_some() {
9244                        self.state.pr_pushed_head = Some(delivery_head.clone());
9245                    }
9246                    // A retry or resume of a run whose branch already has an
9247                    // open pull request adopts it rather than failing on a
9248                    // duplicate. Only this winner branch into this base:
9249                    // `branch_for` derives the name from the run id, so a
9250                    // different run's pull request never matches.
9251                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9252                    let out = match pr_merge_plan(found) {
9253                        PrPlan::Create => {
9254                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9255                                .await
9256                        }
9257                        PrPlan::Adopt { url, title } => {
9258                            self.state
9259                                .event("merge", format!("Pr: adopted open pull request {url}"));
9260                            if self.state.github_text.as_ref().is_some_and(|g| {
9261                                g.resolved
9262                                    && g.choice == Some(GithubTextChoice::Rewrite)
9263                                    && g.rewrite.is_some()
9264                            }) {
9265                                // The owner chose the entire pair, even if the
9266                                // existing title already matches or was edited.
9267                                land::set_pr_text_verbatim(
9268                                    &winner.worktree,
9269                                    &url,
9270                                    &pr.title,
9271                                    &pr.body,
9272                                )
9273                                .await
9274                                .map(|()| url)
9275                            } else {
9276                                if title != pr.title
9277                                    && (!is_review_run(&self.state)
9278                                        || leaked
9279                                            .as_deref()
9280                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9281                                    && let Err(e) =
9282                                        if self.state.github_text.as_ref().is_some_and(|g| {
9283                                            g.resolved
9284                                                && g.title
9285                                                && g.choice == Some(GithubTextChoice::Original)
9286                                        }) {
9287                                            // The owner chose to post this title unmodified.
9288                                            land::set_pr_title_verbatim(
9289                                                &winner.worktree,
9290                                                &url,
9291                                                &pr.title,
9292                                            )
9293                                            .await
9294                                        } else {
9295                                            land::set_pr_title(
9296                                                &mut self.state,
9297                                                &winner.worktree,
9298                                                &url,
9299                                                &pr.title,
9300                                            )
9301                                            .await
9302                                        }
9303                                {
9304                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9305                                    self.state
9306                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9307                                }
9308                                Ok(url)
9309                            }
9310                        }
9311                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9312                    };
9313                    match out {
9314                        Ok(url) => MergeOutcome {
9315                            mode,
9316                            ok: true,
9317                            detail: url,
9318                            empty: false,
9319                        },
9320                        Err(e) => MergeOutcome {
9321                            mode,
9322                            ok: false,
9323                            detail: e.to_string(),
9324                            empty: false,
9325                        },
9326                    }
9327                }
9328            }
9329        };
9330
9331        self.state.status = match (mode, outcome.ok) {
9332            (MergeMode::None, _) => RunStatus::Ready,
9333            (_, true) if outcome.empty => RunStatus::Ready,
9334            (MergeMode::Pr, true) => RunStatus::Ready,
9335            (_, true) => RunStatus::Merged,
9336            (_, false) => RunStatus::Blocked,
9337        };
9338        self.state.event(
9339            "merge",
9340            format!(
9341                "{:?}: {}",
9342                mode,
9343                outcome.detail.lines().next().unwrap_or("")
9344            ),
9345        );
9346        self.state.merge = Some(outcome);
9347        self.state.save()?;
9348
9349        // The PR is open and the run would historically stop here, leaving the
9350        // operator to watch checks, feed review comments back to a fixer, and
9351        // merge. That was done by hand six times in one session before this
9352        // existed. Opt-in, because merging is the one irreversible thing magi
9353        // can do to a repository.
9354        if self.state.config.graph.land
9355            && mode == MergeMode::Pr
9356            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9357        {
9358            self.run_land().await?;
9359        }
9360        // `run_land` may have left `status` at `Landing` - still waiting on
9361        // CI or the owner's approval, not actually settled - so this has to
9362        // read whatever `status` ended up as here, not the `Merged` this
9363        // function set a few lines up.
9364        self.settle_questions();
9365        Ok(())
9366    }
9367
9368    /// Enter `land`.
9369    ///
9370    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9371    /// a resumed run's re-entry. `land::land` itself is what serialises the
9372    /// two git-mutating moments inside the loop — the rebase push and
9373    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9374    /// to hold a lock across the whole call, and doing so would serialise
9375    /// this run's CI wait against a *different* run's land-approval resume
9376    /// in the same repository, which is exactly the "must not wait on
9377    /// another task" property the daemon's slot-freeing exists to give.
9378    async fn run_land(&mut self) -> Result<()> {
9379        crate::pr_head::ensure_owner(&self.state)?;
9380        let url = self
9381            .state
9382            .pr
9383            .as_ref()
9384            .map(|pr| pr.url.clone())
9385            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9386            .unwrap_or_default();
9387        let url = url.lines().next().unwrap_or("").trim().to_owned();
9388        if !url.starts_with("http") {
9389            return Ok(());
9390        }
9391        // A land failure is not a lost run: the work is on a branch and the
9392        // pull request is open, which is exactly where a human takes over.
9393        loop {
9394            match land::land(&mut self.state, &url).await {
9395                Ok(_)
9396                    if self
9397                        .state
9398                        .merge_repairs
9399                        .last()
9400                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9401                {
9402                    if self.repair_merge_review().await? {
9403                        continue;
9404                    }
9405                }
9406                Ok(_) if self.state.resource_block.is_some() => {
9407                    return self.stop_for_resource();
9408                }
9409                Ok(pr) if self.state.parked => {
9410                    // `land` already saved the parked marker; nothing here
9411                    // overrides `status` back to a terminal value while an
9412                    // approval is still outstanding.
9413                    let _ = pr;
9414                }
9415                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9416                    self.state.status = RunStatus::Superseded;
9417                    self.state.save()?;
9418                }
9419                Ok(pr) => {
9420                    self.state.status = match pr.state {
9421                        land::PrLifecycle::Merged => RunStatus::Merged,
9422                        _ => RunStatus::Blocked,
9423                    };
9424                    // Downstream of a confirmed merge only - see
9425                    // `bump::should_release_bump`'s own doc for why this one
9426                    // check covers all three of `land`'s success paths.
9427                    // Best-effort: the run already landed, so a failure here
9428                    // (the decision call, `gh`, `cargo`) is recorded and never
9429                    // turns a landed run into a failed one.
9430                    if bump::should_release_bump(self.state.status)
9431                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9432                    {
9433                        // The event is the run's own record. Not-eligible cases
9434                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9435                        // `Err` is a bump that was tried and failed:
9436                        // `after_merge` itself raises the operator notice for
9437                        // that, whether or not a release PR exists yet.
9438                        self.state
9439                            .event("bump", format!("release bump skipped: {e:#}"));
9440                    }
9441                    // Independent of the bump, and best-effort in the same way:
9442                    // findings the merge left open become follow-up tasks.
9443                    if self.state.status == RunStatus::Merged {
9444                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9445                    }
9446                    self.state.save()?;
9447                }
9448                Err(e) => {
9449                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9450                        return Err(e);
9451                    }
9452                    self.state.status = RunStatus::Blocked;
9453                    self.state.event("land", format!("gave up: {e}"));
9454                    self.state.save()?;
9455                }
9456            }
9457            break;
9458        }
9459        Ok(())
9460    }
9461
9462    /// One repair cycle stays under the existing driver/task claim. An
9463    /// interrupted fixer is rolled back and held rather than run twice.
9464    async fn repair_merge_review(&mut self) -> Result<bool> {
9465        use crate::run::MergeRepairPhase as Phase;
9466        crate::pr_head::ensure_owner(&self.state)?;
9467        let i = self.state.merge_repairs.len() - 1;
9468        let request = self.state.merge_repairs[i].clone();
9469        if request.phase == Phase::Fixing {
9470            let restored = match self.state.winner().cloned() {
9471                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9472                None => Err(anyhow::anyhow!("repair has no winning branch")),
9473            };
9474            let why = match restored {
9475                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9476                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9477                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9478            };
9479            self.hold_merge_repair(i, why)?;
9480            return Ok(false);
9481        }
9482        // A completed repair gate is a publication checkpoint. On reentry,
9483        // reject external changes before the ordinary gate path can replace
9484        // that evidence by verifying a new head. Requested repairs still
9485        // start their own review generation below.
9486        if request.phase == Phase::Reviewing && self.state.gate_ran {
9487            let winner = self
9488                .state
9489                .winner()
9490                .context("repair has no winning branch")?;
9491            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9492            if self.state.gate_head.as_deref() != Some(head.as_str())
9493                || !git::is_clean(&winner.worktree).await?
9494            {
9495                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9496                return Ok(false);
9497            }
9498        }
9499        let failure = if i >= self.state.config.graph.land_rounds
9500            || self.state.config.graph.review_rounds == 0
9501        {
9502            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9503        } else {
9504            None
9505        };
9506        if let Some(why) = failure {
9507            self.hold_merge_repair(i, why)?;
9508            return Ok(false);
9509        }
9510        if request.phase == Phase::Requested {
9511            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9512                Ok(pr) => pr,
9513                Err(e) => {
9514                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9515                    return Ok(false);
9516                }
9517            };
9518            self.state.merge_repairs[i].phase = Phase::Fixing;
9519            self.state.land_approval = None;
9520            self.state.land_armed_head = None;
9521            self.state.save()?;
9522            // Serialize the exact records, including finding ids/details,
9523            // reconsideration reasons, final votes and verification output.
9524            let evidence = serde_json::to_string_pretty(&request)?;
9525            let reason = format!(
9526                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9527            );
9528            let budget = self.state.config.graph.land_rounds;
9529            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9530            match result {
9531                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9532                Ok(land::Fixed::Deferred) => {
9533                    // An unavailable fixer did not spend this repair request.
9534                    // Restore partial fallback work before retrying it, as for
9535                    // an interrupted repair, without closing the request.
9536                    let worktree = self
9537                        .state
9538                        .winner()
9539                        .context("repair has no winning branch")?
9540                        .worktree
9541                        .clone();
9542                    self.graft_restore(&worktree, &request.head).await?;
9543                    self.state.merge_repairs[i].phase = Phase::Requested;
9544                    self.stop_for_resource()?;
9545                    return Ok(false);
9546                }
9547                Ok(land::Fixed::Failed(why)) => {
9548                    self.hold_merge_repair(
9549                        i,
9550                        format!(
9551                            "repair failed: {why}; inspect the branch and run an explicit review"
9552                        ),
9553                    )?;
9554                    return Ok(false);
9555                }
9556                Err(e) => {
9557                    self.hold_merge_repair(
9558                        i,
9559                        format!(
9560                            "repair failed: {e:#}; inspect the branch and run an explicit review"
9561                        ),
9562                    )?;
9563                    return Ok(false);
9564                }
9565            }
9566            // Even no change gets a real new reviewer wave. The previous
9567            // evidence remains in the request; a former clean/concluded round
9568            // cannot short-circuit the new review or gate.
9569            self.begin_merge_repair_review(i)?;
9570        }
9571        self.review_loop().await?;
9572        self.gate().await?;
9573        if self.resource_halt()? || self.state.parked {
9574            return Ok(false);
9575        }
9576        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9577            != Some(RunStatus::Gating)
9578            || !self.state.gate_allows_progress()
9579        {
9580            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9581            return Ok(false);
9582        }
9583        // Review and gate may have added commits after the initial repair.
9584        // Publish exactly their final head to the existing PR, with a lease.
9585        let winner = self
9586            .state
9587            .winner()
9588            .cloned()
9589            .context("repair has no winning branch")?;
9590        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9591        if self.state.gate_head.as_deref() != Some(after.as_str())
9592            || !git::is_clean(&winner.worktree).await?
9593        {
9594            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9595            return Ok(false);
9596        }
9597        let expected = self
9598            .state
9599            .pr_pushed_head
9600            .as_deref()
9601            .unwrap_or(&request.head)
9602            .to_owned();
9603        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9604        // Push the checked commit itself: a local ref moving after the check
9605        // must not change which commit the forge receives.
9606        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9607            return Ok(false);
9608        }
9609        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9610        let refspec = format!("{after}:refs/heads/{public}");
9611        let push = git::git_raw(
9612            &winner.worktree,
9613            &["push", &lease, &self.state.config.merge.remote, &refspec],
9614        )
9615        .await;
9616        match push {
9617            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9618            Ok(out) => {
9619                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9620                return Ok(false);
9621            }
9622            Err(e) => {
9623                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9624                return Ok(false);
9625            }
9626        }
9627        self.state.merge_repairs[i].phase = Phase::Complete;
9628        self.state.status = RunStatus::Landing;
9629        self.state.event(
9630            "repair",
9631            "new review and gate completed; fresh owner merge decision required",
9632        );
9633        self.state.save()?;
9634        Ok(true)
9635    }
9636
9637    fn merge_repair_artifacts(&self) -> PathBuf {
9638        let artifacts = agent::artifacts_dir(&self.state.dir());
9639        match self
9640            .state
9641            .merge_repairs
9642            .last()
9643            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9644        {
9645            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9646            None => artifacts,
9647        }
9648    }
9649
9650    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9651        if let Some(e2e) = self.state.head_e2e.take() {
9652            self.state
9653                .verification_history
9654                .push(crate::run::VerificationEvidence {
9655                    superseded_by: None,
9656                    archived_at: Some(Timestamp::now()),
9657                    fixes_spent: Some(self.state.gate_fixes.len()),
9658                    gate_head: None,
9659                    gate_ran: false,
9660                    gate: Vec::new(),
9661                    e2e: Some(e2e),
9662                });
9663        }
9664        self.state.reviews.clear();
9665        self.state.gate.clear();
9666        self.state.gate_ran = false;
9667        self.state.gate_head = None;
9668        self.state.gate_fixes.clear();
9669        self.state.gate_repair_e2e = None;
9670        self.state.pre_gate.clear();
9671        self.state.pre_gate_commit = None;
9672        self.state.contested_handoff = None;
9673        self.state.land_reruns = None;
9674        self.state.land_approval = None;
9675        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9676        self.state.status = RunStatus::Reviewing;
9677        self.state.save()
9678    }
9679
9680    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9681        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9682        self.state.merge_repairs[index].failure = Some(why.clone());
9683        self.state.status = RunStatus::Blocked;
9684        self.state.event("repair", why);
9685        self.state.save()
9686    }
9687
9688    // -------------------------------------------------------------- helpers
9689
9690    /// Fetch or create a seat, keeping its conversation across nodes.
9691    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9692        if let Some(existing) = self.state.seats.get(key)
9693            && existing.agent == agent
9694        {
9695            return existing.clone();
9696        }
9697        // A seat that changes agent mints its session id from the agent too,
9698        // like a handover: the old agent's uuid is already taken by the CLI.
9699        let fresh = if self.state.seats.contains_key(key) {
9700            handover_seat(key, agent, self.state.next_seat_seed())
9701        } else {
9702            SeatState::new(key, agent, self.state.seed)
9703        };
9704        self.state.seats.insert(key.to_owned(), fresh.clone());
9705        fresh
9706    }
9707
9708    /// The agent now holding seat `key`: `spec`, unless a handover moved the
9709    /// seat to another roster agent, in which case that agent. Nodes that
9710    /// continue a seat's conversation (deliberation, the votes, a reviewer's
9711    /// reconsideration) must keep talking to whoever answered it, not slip
9712    /// back to the agent that failed it.
9713    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9714        match self.state.seats.get(key) {
9715            Some(s) if s.agent != spec.id => {
9716                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9717            }
9718            _ => spec,
9719        }
9720    }
9721
9722    /// A candidate rendered for judging, with the leak policy applied.
9723    fn view(&self, c: &Candidate) -> CandidateView {
9724        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9725            .unwrap_or_default();
9726        let (patch, _) = blind::sanitize_patch(
9727            &format!("candidate {} patch", c.label),
9728            &raw,
9729            &self.state.config.blind,
9730        );
9731        CandidateView {
9732            label: c.label,
9733            branch: c.branch.clone(),
9734            summary: c.summary.clone(),
9735            stat: c.stat.clone(),
9736            patch,
9737        }
9738    }
9739
9740    /// The full candidate set as prompt text, for seats with no live session.
9741    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9742        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9743        prompt::judge(
9744            "(see above)",
9745            &views,
9746            self.roles.judges.len(),
9747            base_short,
9748            "en",
9749        )
9750    }
9751
9752    /// The final-vote prompt with everything a seat that has no session of its
9753    /// own needs: the candidates, the seat's own ranking and reasons, and the
9754    /// anonymised deliberation it took part in (only when there was one, so a
9755    /// handed-over seat never sees more than the seat it replaces did). The
9756    /// `Final vote` heading stays first.
9757    fn vote_prompt_full(
9758        &self,
9759        j: usize,
9760        labels: &[char],
9761        language: &str,
9762        candidates: &[Candidate],
9763        base_short: &str,
9764    ) -> String {
9765        let mut text = format!(
9766            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9767            prompt::final_vote(labels, language),
9768            self.state.instruction,
9769            self.candidate_block(candidates, base_short)
9770        );
9771        if let Some(own) = self
9772            .state
9773            .judgements
9774            .get(j)
9775            .filter(|r| !r.ranking.is_empty())
9776        {
9777            let reasons = own
9778                .reasons
9779                .iter()
9780                .map(|(k, v)| format!("- {k}: {v}"))
9781                .collect::<Vec<_>>()
9782                .join("\n");
9783            text.push_str(&format!(
9784                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9785                own.ranking.iter().collect::<String>(),
9786                if reasons.is_empty() {
9787                    ""
9788                } else {
9789                    ", because:\n"
9790                }
9791            ));
9792        }
9793        if !self.state.deliberation.is_empty() {
9794            text.push_str("\n# What was argued before this vote\n");
9795            for t in self.transcript(&[], j) {
9796                text.push_str(&format!(
9797                    "\n## {}{}\n\n{}\n",
9798                    t.who,
9799                    if t.is_self { " (you)" } else { "" },
9800                    t.body.trim()
9801                ));
9802            }
9803        }
9804        text
9805    }
9806
9807    /// Anonymised transcript for judge `self_idx`.
9808    ///
9809    /// The initial rankings are always the opening statements. Seeding them
9810    /// only when no turn had been taken yet meant every judge after the first
9811    /// argued against a single voice instead of against the actual split — the
9812    /// disagreement is the information, so it is always on the table.
9813    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9814        let mut turns = Vec::new();
9815        for j in &self.state.judgements {
9816            if j.ranking.is_empty() {
9817                continue;
9818            }
9819            let reasons = j
9820                .reasons
9821                .iter()
9822                .map(|(k, v)| format!("- {k}: {v}"))
9823                .collect::<Vec<_>>()
9824                .join("\n");
9825            turns.push(Turn {
9826                who: format!("Judge {} (opening ranking)", j.judge),
9827                is_self: j.judge == self_idx + 1,
9828                body: format!(
9829                    "Ranked {}{}{reasons}",
9830                    j.ranking.iter().collect::<String>(),
9831                    if reasons.is_empty() {
9832                        ""
9833                    } else {
9834                        ", because:\n"
9835                    }
9836                ),
9837            });
9838        }
9839        for t in self
9840            .state
9841            .deliberation
9842            .iter()
9843            .flat_map(|r| r.turns.iter())
9844            .chain(current)
9845        {
9846            turns.push(Turn {
9847                who: format!("Judge {}", t.judge),
9848                is_self: t.judge == self_idx + 1,
9849                body: t.body.clone(),
9850            });
9851        }
9852        turns
9853    }
9854}
9855
9856/// Does this seat still hold the context a follow-up prompt would rely on?
9857fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9858    agent::has_session(spec.kind, seat, sessions)
9859}
9860
9861/// The next entry in `roster` after `start`, never wrapping back to the
9862/// front, whose id is not in `tried` yet.
9863///
9864/// Starts one past `start` rather than at the front of `roster`: `start` is
9865/// the seat's own original position, and a seat whose candidate slot already
9866/// sits on the roster's second entry must fall through to the third next, not
9867/// restart at the first — which is very likely a different candidate's own
9868/// agent already. Never wraps back past `start`, for the same reason: an
9869/// entry earlier in the roster than the seat's own position is almost
9870/// certainly some *other* candidate slot's own agent, and once the tail of
9871/// the roster is exhausted there are no more untried agents for *this* seat
9872/// to fall through to — the caller's fallback chain ends there, exactly as
9873/// "no further untried agents remain in the list for that seat" asks for.
9874///
9875/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
9876/// the same id twice (an operator's `roles.implementers` typo, or a
9877/// `[[agents]]` list reused across roles) must not let
9878/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
9879/// over `roster` either finds an untried id or runs out, so this always
9880/// terminates regardless of duplicates.
9881fn next_untried_in_roster<'a>(
9882    roster: &'a [AgentSpec],
9883    start: usize,
9884    tried: &BTreeSet<String>,
9885) -> Option<&'a AgentSpec> {
9886    roster
9887        .get(start + 1..)?
9888        .iter()
9889        .find(|s| !tried.contains(&s.id))
9890}
9891
9892/// The successor for a seat, shared by all four seat kinds (implement, judge,
9893/// review, advise). `others` holds the ids that *currently* occupy the other
9894/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
9895/// sees them), so roster entries beyond the seat count act as spares: a failed
9896/// seat goes to an agent no other seat holds whenever the roster permits.
9897///
9898/// `carried` is `Some` only for the review loop's carried failure history.
9899/// Order: (1) forward from `start`, never wrapping, untried, not a carried
9900/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
9901/// whole roster: untried and not another seat's occupant; (3) the plain walk
9902/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
9903/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
9904/// a better outcome than an empty seat, and it keeps the answer to "is there
9905/// a successor at all" exactly what it was before occupants were considered,
9906/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
9907/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
9908/// carried failure is a cheaper bet than doubling an agent on the panel, and
9909/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
9910/// the same round are handled one at a time, so a seat later in the batch
9911/// sees an earlier one's new occupant but not yet its own freed agent.
9912fn pick_successor<'a>(
9913    roster: &'a [AgentSpec],
9914    start: usize,
9915    tried: &BTreeSet<String>,
9916    carried: Option<&BTreeSet<String>>,
9917    others: &BTreeSet<String>,
9918) -> Option<&'a AgentSpec> {
9919    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9920    roster
9921        .get(start + 1..)
9922        .and_then(|tail| {
9923            tail.iter()
9924                .filter(free)
9925                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9926        })
9927        .or_else(|| {
9928            carried
9929                .is_some()
9930                .then(|| roster.iter().find(free))
9931                .flatten()
9932        })
9933        .or_else(|| match carried {
9934            Some(c) => next_for_seat(roster, start, tried, c),
9935            None => next_untried_in_roster(roster, start, tried),
9936        })
9937}
9938
9939/// The next agent for a seat that carries its failure history across rounds
9940/// (the review loop). `round_tried` is this round's own bound and starts
9941/// empty every round; `carried_failed` only decides priority.
9942///
9943/// First: an id walking forward from `start`, never wrapping, that is neither
9944/// tried this round nor failed in an earlier one. Only when that is exhausted
9945/// does it rescue: the first roster id (in roster order, so this one *does*
9946/// look before `start`) not yet tried this round, which is by then a carried
9947/// failure. Each id is rescued at most once per round, so it cannot loop.
9948fn next_for_seat<'a>(
9949    roster: &'a [AgentSpec],
9950    start: usize,
9951    round_tried: &BTreeSet<String>,
9952    carried_failed: &BTreeSet<String>,
9953) -> Option<&'a AgentSpec> {
9954    roster
9955        .get(start + 1..)?
9956        .iter()
9957        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9958        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9959}
9960
9961/// Where a reviewer seat starts a round: the agent that last answered it when
9962/// it is still on the roster and not marked failed, else the spec's own agent
9963/// unless it failed, else the next roster agent that has not failed, else the
9964/// spec's own agent again (the whole roster failed). Ids no longer on the
9965/// roster are ignored. An empty roster has no handover, so the spec stands.
9966fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9967    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9968        return spec;
9969    };
9970    let ok = |id: &str| !h.failed.contains(id);
9971    if let Some(last) = h.last_ok.as_deref()
9972        && ok(last)
9973        && let Some(s) = roster.iter().find(|s| s.id == last)
9974    {
9975        return s.clone();
9976    }
9977    if ok(&spec.id) {
9978        return spec;
9979    }
9980    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9981    roster
9982        .get(from + 1..)
9983        .into_iter()
9984        .flatten()
9985        .chain(roster.iter())
9986        .find(|s| ok(&s.id))
9987        .cloned()
9988        .unwrap_or(spec)
9989}
9990
9991/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
9992/// seed so a CLI that mints its session id up front (`--session-id`) never
9993/// reuses the uuid the previous agent already opened under the same seat key.
9994pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9995    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9996}
9997
9998/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
9999/// so the classifier below and the code that builds the message cannot drift.
10000const TIMED_OUT: &str = "timed out";
10001
10002impl FailClass {
10003    /// `None` for an answer; otherwise how the turn failed.
10004    fn of(out: &AgentOutcome) -> Option<Self> {
10005        match out {
10006            AgentOutcome::Ok(_) => None,
10007            AgentOutcome::Quota(_) => Some(Self::Quota),
10008            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10009            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10010            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10011            // Not a failure of the agent. Callers check for it first; `None`
10012            // here means "no class to hand over on".
10013            AgentOutcome::ResourceBlocked(_) => None,
10014        }
10015    }
10016
10017    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10018    fn stem_word(&self) -> &'static str {
10019        match self {
10020            Self::Quota => "quota",
10021            _ => "handover",
10022        }
10023    }
10024}
10025
10026/// The message's first line with its variable parts removed — digit runs and
10027/// path-like tokens — so "exit code 2" and "exit code 7" read
10028/// as one kind of failure.
10029fn failure_signature(msg: &str) -> String {
10030    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10031    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10032    // the historical machine key while the process diagnosis uses readable prose.
10033    if let Some(code) = line
10034        .strip_prefix("process exited with no usable output (exit code ")
10035        .and_then(|s| s.strip_suffix(')'))
10036        .and_then(|s| s.parse::<i32>().ok())
10037    {
10038        line = format!("exited with some({code}) and no usable output");
10039    } else if matches!(
10040        line.as_str(),
10041        "process exited with no usable output (no exit code reported)"
10042            | "process exited with no usable output (exit code unavailable)"
10043    ) {
10044        line = "exited with none and no usable output".to_owned();
10045    }
10046    let mut out = Vec::new();
10047    for word in line.split_whitespace() {
10048        if word.contains('/') || word.contains('\\') {
10049            out.push("<path>".to_owned());
10050            continue;
10051        }
10052        let mut w = String::new();
10053        let mut in_digits = false;
10054        for c in word.chars() {
10055            if c.is_ascii_digit() {
10056                if !in_digits {
10057                    w.push('#');
10058                }
10059                in_digits = true;
10060            } else {
10061                in_digits = false;
10062                w.push(c);
10063            }
10064        }
10065        out.push(w);
10066    }
10067    out.join(" ").chars().take(120).collect()
10068}
10069
10070/// Whether a seat that just failed with `cur` may go to the next roster agent.
10071/// A quota or a timeout always may. Any other failure may not when the agent
10072/// before it failed the same way: an error the prompt causes would otherwise
10073/// walk the whole roster. `prev` is the class of the immediately preceding
10074/// agent's failure, so a quota or timeout in between breaks the run of
10075/// identical failures by itself.
10076fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10077    match cur {
10078        FailClass::Quota | FailClass::Timeout => true,
10079        FailClass::Other(_) => prev != Some(cur),
10080    }
10081}
10082
10083/// A short human reason for a failed outcome, for the handover record.
10084fn fail_reason(out: &AgentOutcome) -> String {
10085    match out {
10086        AgentOutcome::Ok(_) => String::new(),
10087        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10088        AgentOutcome::Dropped(o) => format!(
10089            "the CLI dropped the stream ({})",
10090            o.dropped
10091                .as_ref()
10092                .map(|d| d.why.as_str())
10093                .unwrap_or("it ended without delivering its answer")
10094        ),
10095        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10096            e.lines().next().unwrap_or("").chars().take(160).collect()
10097        }
10098    }
10099}
10100
10101/// Note one handover in the run: the structured record and, in the timeline,
10102/// the sentence a person reads. A quota keeps the wording it always had.
10103pub(crate) fn record_handover(
10104    state: &mut RunState,
10105    node: &str,
10106    seat: &str,
10107    from: &str,
10108    to: &str,
10109    class: &FailClass,
10110    reason: &str,
10111) {
10112    let message = if *class == FailClass::Quota {
10113        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10114    } else {
10115        format!("{seat}: handed over {from} -> {to} ({reason})")
10116    };
10117    state.event(node, message);
10118    state.handovers.push(Handover {
10119        at: Timestamp::now(),
10120        node: node.to_owned(),
10121        seat: seat.to_owned(),
10122        from: from.to_owned(),
10123        to: to.to_owned(),
10124        reason: reason.to_owned(),
10125    });
10126}
10127
10128/// Did this reply report running a command whose own CLI never confirmed an
10129/// exit status?
10130///
10131/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10132/// command *finished* (see that type's own doc), so this can only be `true`
10133/// for a command whose completion event carried no readable exit code — not
10134/// for one that simply is not mentioned at all. That is the one signal this
10135/// crate can read, from the same record `state.jobs` renders, about a reply
10136/// standing next to work its own CLI cannot vouch for finishing; it is
10137/// deliberately not a check on the exit code's *value* (a fixer legitimately
10138/// runs a command that fails mid-iteration before it succeeds) and not a
10139/// guess at a command still running in the background (which emits no event
10140/// at all, and so leaves no evidence here to find).
10141fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10142    commands.iter().any(|c| c.exit_code.is_none())
10143}
10144
10145/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10146/// trusted as a verified no-op — the adoption guard's own text-level half.
10147///
10148/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10149/// timed out): a marker only earns the benefit of the doubt from a turn the
10150/// CLI itself vouches for finishing properly, the same house style
10151/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10152/// *fix* report to for `commands`. A candidate that timed out, exited
10153/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10154/// is, whatever prose it wrote — this returns `None` before it ever looks at
10155/// `text`. The remaining guards (the tree really is empty, the evidence is
10156/// non-empty) are the caller's: this only reads what the reply *claimed*.
10157fn verified_noop_claim(
10158    usable: bool,
10159    commands: &[agent::CommandEvidence],
10160    text: &str,
10161) -> Option<String> {
10162    (usable && !has_unconfirmed_command(commands))
10163        .then(|| verdict::verified_noop(text))
10164        .flatten()
10165}
10166
10167fn short(commit: &str) -> String {
10168    commit.chars().take(7).collect()
10169}
10170
10171fn make_executable(path: &Path) -> Result<()> {
10172    #[cfg(unix)]
10173    {
10174        use std::os::unix::fs::PermissionsExt as _;
10175        let mut perms = std::fs::metadata(path)?.permissions();
10176        perms.set_mode(0o755);
10177        std::fs::set_permissions(path, perms)?;
10178    }
10179    #[cfg(not(unix))]
10180    {
10181        let _ = path;
10182    }
10183    Ok(())
10184}
10185
10186/// What every seat in one batch shares: where the answers are attributed, the
10187/// prompt overlay they inherit, and the build cache they are told to use.
10188///
10189/// A struct rather than four more parameters: `wave` also needs the run's
10190/// state (to record who is answering right now) and the attempt number, and
10191/// eight positional arguments is both unreadable and a clippy error.
10192struct WaveCtx<'a> {
10193    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10194    /// paid for it.
10195    run: &'a str,
10196    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10197    node: &'a str,
10198    prompts: &'a Prompts,
10199    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10200    cache: Option<&'a Path>,
10201    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10202    /// `JobRecord::round`. `None` for every other node.
10203    round: Option<usize>,
10204    /// Carry each seat's failed-agent history across waves (the review loop
10205    /// only): start-of-round priority and handover choice read
10206    /// [`RunState::seat_history`], and every answer or failure writes it.
10207    carry_seats: bool,
10208    /// The run's park / interrupt requests, so a wait for a shared resource
10209    /// can be abandoned the moment the operator asks to park.
10210    pauses: Option<(&'a Pause, &'a Pause)>,
10211}
10212
10213/// Shared scheduling policy for operations and spawned seat invocations.
10214fn repository_wait(
10215    state: &RunState,
10216    pauses: Option<(&Pause, &Pause)>,
10217) -> crate::repo_lock::WaitPolicy {
10218    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10219    let inherited = crate::repo_lock::WaitPolicy::current();
10220    crate::repo_lock::WaitPolicy::new(
10221        Duration::from_secs(state.config.graph.timeout_resource_wait),
10222        move || {
10223            inherited.cancelled()
10224                || pauses
10225                    .as_ref()
10226                    .is_some_and(|(a, b)| a.parked() || b.parked())
10227        },
10228    )
10229}
10230
10231fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10232    if error.is::<crate::repo_lock::ResourceBlocked>() {
10233        state.resource_block = Some(crate::run::ResourceBlock {
10234            cause: crate::run::ResourceBlockCause::Other,
10235            node: "repository".into(),
10236            kind: if error.to_string().contains("disk pressure") {
10237                "disk pressure"
10238            } else {
10239                "repository coordination"
10240            }
10241            .into(),
10242            detail: error.to_string(),
10243            at: Timestamp::now(),
10244        });
10245        state.event("resource", error.to_string());
10246    }
10247}
10248
10249async fn coordinate_operation(
10250    state: &mut RunState,
10251    cwd: &Path,
10252    wait: &crate::repo_lock::WaitPolicy,
10253) -> Result<Option<crate::repo_lock::SeatLease>> {
10254    let result =
10255        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10256            .await;
10257    if let Err(e) = &result {
10258        record_repository_block(state, e);
10259    }
10260    result
10261}
10262
10263/// Run one job, honouring the parallelism budget.
10264async fn run_one(
10265    job: SeatJob,
10266    sem: Arc<Semaphore>,
10267    ctx: &WaveCtx<'_>,
10268    state: &mut RunState,
10269    attempt: usize,
10270) -> (SeatState, AgentOutcome) {
10271    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10272        .await
10273        .pop()
10274        .expect("one job in, one result out");
10275    (seat, out)
10276}
10277
10278/// Run every job concurrently, capped by the semaphore, preserving order.
10279///
10280/// Every seat in the batch is recorded into [`RunState::active`] before the
10281/// wave starts and cleared as each answer lands, so the run's own record says
10282/// who is still being waited on rather than only who finished.
10283async fn wave(
10284    jobs: Vec<SeatJob>,
10285    sem: Arc<Semaphore>,
10286    ctx: &WaveCtx<'_>,
10287    state: &mut RunState,
10288    attempt: usize,
10289) -> Vec<(usize, SeatState, AgentOutcome)> {
10290    let WaveCtx {
10291        run,
10292        node,
10293        prompts,
10294        cache,
10295        round,
10296        carry_seats: _,
10297        pauses,
10298    } = *ctx;
10299    for job in &jobs {
10300        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10301    }
10302    if let Err(e) = state.save() {
10303        // A failed persist of "who is answering right now" must not abort the
10304        // wave: the seats are already being asked, and the alternative is
10305        // losing the answers to save a status line nobody may even be
10306        // watching.
10307        tracing::warn!("could not persist in-progress seats: {e:#}");
10308    }
10309    // Hold the shared build cache's lease for the whole batch, not per job:
10310    // several candidates (an implement wave) or a fixer legitimately share
10311    // one cache concurrently within this run, and that stays untouched — a
10312    // single lease taken once for the whole wave and released once it is
10313    // done is what stops a *different* borrower (another run's own wave, its
10314    // e2e/gate, a human's `magi review`) from interleaving a build into the
10315    // same directory while this one is in flight. Best-effort, not
10316    // all-or-nothing: a wave that cannot get the lease within its own
10317    // longest job's budget still runs — an hour of paid implementer calls is
10318    // not thrown away over cache contention — but every write-allowed seat
10319    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10320    // below), the same fallback a read-only seat always gets, rather than
10321    // building into a directory this run was never granted. The identity
10322    // record is still invalidated below either way, so the next tracked
10323    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10324    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10325    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10326    // never borrowed from a job's agent budget (and never charged to it either,
10327    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10328    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10329    let cache_guard = if let Some(cache_dir) = cache {
10330        if jobs_had_a_writer {
10331            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10332            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10333            match acquire_cache_lease(
10334                state,
10335                cache_dir,
10336                &owner,
10337                resource_limit,
10338                node,
10339                node,
10340                &cancel,
10341            )
10342            .await
10343            {
10344                Ok(Some(g)) => Some(g),
10345                // Parking: leave without spawning anything. The run parks at
10346                // the next boundary and the seats are asked afresh on resume.
10347                Ok(None) => {
10348                    if let Some(block) = &mut state.resource_block {
10349                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10350                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10351                        };
10352                    }
10353                    return block_wave(
10354                        jobs,
10355                        state,
10356                        node,
10357                        attempt,
10358                        "parked while waiting for the shared build cache",
10359                    );
10360                }
10361                // An unreadable lease is not contention: keep the old
10362                // fallback (no shared cache for this wave's seats).
10363                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10364                Err(e) => {
10365                    // Writing seats may not build into a cache this run does
10366                    // not hold, and an agent that cannot be given one is not
10367                    // asked at all: the run stops resumably instead.
10368                    let why = format!("{e:#}");
10369                    state.event(
10370                        "resource",
10371                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10372                    );
10373                    return block_wave(jobs, state, node, attempt, &why);
10374                }
10375            }
10376        } else {
10377            None
10378        }
10379    } else {
10380        None
10381    };
10382    // Every job keeps its whole budget: the wait above was not the agent's.
10383    // Restart the seats' clocks so "started" means "being spawned now".
10384    state.seats_spawning(node, attempt);
10385    if let Err(e) = state.save() {
10386        tracing::warn!("could not persist the start of a wave: {e:#}");
10387    }
10388    // Queueing behind this wave's own seats for a parallelism slot is normal
10389    // (each of them is bounded by its own agent timeout); only a wait beyond
10390    // all of them plus the resource limit means the slots never came back.
10391    let slot_limit = resource_limit
10392        + jobs
10393            .iter()
10394            .map(|j| j.timeout)
10395            .max()
10396            .unwrap_or_default()
10397            .saturating_mul(jobs.len() as u32);
10398    let mut set = tokio::task::JoinSet::new();
10399    // Seats announce themselves here the moment they hold a parallelism
10400    // permit, i.e. just before their CLI is spawned.
10401    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10402    let overlay = prompts.overlay(node);
10403    let repository_wait = repository_wait(state, pauses);
10404    for (i, mut job) in jobs.into_iter().enumerate() {
10405        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10406        if cache.is_some() {
10407            job.prompt.push('\n');
10408            job.prompt
10409                .push_str(&prompt::build_cache_note(node, job.allow_write));
10410        }
10411        let sem = Arc::clone(&sem);
10412        let run = run.to_owned();
10413        let node = node.to_owned();
10414        // Only implementers were told about the task's attachments, so only
10415        // their seats get the directory widened for reading.
10416        let attachments = if node == "implement" {
10417            state.attachments.clone()
10418        } else {
10419            Vec::new()
10420        };
10421        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10422        // `prompt::build_cache_note`'s doc for why setting it anyway is
10423        // exactly how a sandboxed reviewer's write refusal got reported as a
10424        // defect in the patch, not a property of its own seat. And a
10425        // write-allowed one is handed it only when the lease above was
10426        // actually acquired: a wave that could not get it (`cache_guard` is
10427        // `None`, see its own comment) must not send seats to build into a
10428        // directory this run does not hold - that is the exact concurrent,
10429        // unmanaged-write race this module exists to prevent, not something
10430        // "proceeding anyway" is allowed to reintroduce.
10431        let cache = cache
10432            .filter(|_| job.allow_write && cache_guard.is_some())
10433            .map(Path::to_path_buf);
10434        let spawned_tx = spawned_tx.clone();
10435        let repository_wait = repository_wait.clone();
10436        let coordination = state.config.coordination.clone();
10437        let fetched_base = state.fetched_base.clone();
10438        set.spawn(async move {
10439            // A wait for a parallelism slot is scheduling, not the agent: it
10440            // is bounded by its own limit and ends as a resource block, never
10441            // as a timeout of the agent that was not started.
10442            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10443                Ok(p) => p,
10444                Err(_) => {
10445                    return (
10446                        i,
10447                        job.seat,
10448                        AgentOutcome::ResourceBlocked(format!(
10449                            "no parallel slot became free within {}s",
10450                            slot_limit.as_secs()
10451                        )),
10452                        false,
10453                    );
10454                }
10455            };
10456            spawned_tx.send(job.seat.key.clone()).ok();
10457            let mut seat = job.seat;
10458            let out = repository_wait
10459                .scope(agent::invoke(
10460                    &job.spec,
10461                    &mut seat,
10462                    &Invocation {
10463                        coordination: coordination.as_ref(),
10464                        fetched_base: fetched_base.as_ref(),
10465                        cwd: &job.cwd,
10466                        prompt: &job.prompt,
10467                        timeout: job.timeout,
10468                        allow_write: job.allow_write,
10469                        sessions: job.sessions,
10470                        artifacts: &job.artifacts,
10471                        stem: &job.stem,
10472                        run: &run,
10473                        node: &node,
10474                        cache_dir: cache.as_deref(),
10475                        attachments: &attachments,
10476                        writable: &[],
10477                    },
10478                ))
10479                .await;
10480            let interrupted = out
10481                .as_ref()
10482                .err()
10483                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10484            let out = match out {
10485                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10486                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10487                // Billed work the CLI failed to hand over is not an ordinary
10488                // failure, but its text is the CLI's raw error JSON, not an
10489                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10490                // read it as one by forgetting to check. `usable()` is always
10491                // false here (dropped implies an empty response), so this has
10492                // to be checked before the catch-all `Failed` below or the
10493                // one shape this exists for is lost with the rest.
10494                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10495                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10496                Ok(o) => AgentOutcome::Failed(format!(
10497                    "process exited with no usable output ({})",
10498                    crate::notices::exit_status(o.exit_code)
10499                )),
10500                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10501                    AgentOutcome::ResourceBlocked(e.to_string())
10502                }
10503                Err(e) => AgentOutcome::Failed(e.to_string()),
10504            };
10505            (i, seat, out, interrupted)
10506        });
10507    }
10508    drop(spawned_tx);
10509    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10510    loop {
10511        let joined = tokio::select! {
10512            Some(key) = spawned_rx.recv() => {
10513                // The clock of this seat starts now, not when it queued.
10514                state.seat_spawned(&key);
10515                if let Err(e) = state.save() {
10516                    tracing::warn!("could not persist a seat's start: {e:#}");
10517                }
10518                continue;
10519            }
10520            joined = set.join_next() => joined,
10521        };
10522        let Some(joined) = joined else {
10523            break;
10524        };
10525        let (i, seat, out, interrupted) = match joined {
10526            Ok(v) => v,
10527            // No seat to clear: a panicked task never reported which one it
10528            // was. The defensive sweep below this loop is what stops that
10529            // seat's `active` entry from surviving forever.
10530            Err(e) => {
10531                tracing::error!("agent task panicked: {e}");
10532                continue;
10533            }
10534        };
10535        state.seat_finished(&seat.key);
10536        if interrupted
10537            && node == "implement"
10538            && let Some(label) = state
10539                .candidates
10540                .iter()
10541                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10542                .map(|candidate| candidate.label)
10543        {
10544            state.candidate_preservation.insert(
10545                label,
10546                "interrupted implementation awaiting persistence".into(),
10547            );
10548            state.interrupted_implementations.insert(label);
10549        }
10550        if let AgentOutcome::ResourceBlocked(why) = &out
10551            && state.resource_block.is_none()
10552        {
10553            state.resource_block = Some(crate::run::ResourceBlock {
10554                cause: crate::run::ResourceBlockCause::Other,
10555                node: node.to_owned(),
10556                kind: if why.contains("disk pressure") {
10557                    "disk pressure"
10558                } else if why.contains("repository coordination") {
10559                    "repository coordination"
10560                } else {
10561                    "parallel slot"
10562                }
10563                .to_owned(),
10564                detail: why.clone(),
10565                at: Timestamp::now(),
10566            });
10567            let activity = if interrupted {
10568                "agent execution interrupted"
10569            } else {
10570                "no agent was started"
10571            };
10572            state.event("resource", format!("{node}: {activity} - {why}"));
10573        }
10574        record_jobs(state, node, round, &seat.key, &out);
10575        if let Err(e) = state.save() {
10576            tracing::warn!("could not persist a seat's completion: {e:#}");
10577        }
10578        if collected.len() <= i {
10579            collected.resize_with(i + 1, || None);
10580        }
10581        collected[i] = Some((i, seat, out));
10582    }
10583    // Belt-and-braces for the panic branch above: every seat this exact batch
10584    // started shares this `(node, attempt)` pair, and every seat that finished
10585    // normally already cleared itself, so anything left tagged with it here
10586    // can only be a panicked task's leftover. Cleared unconditionally rather
10587    // than left to read as still answering forever.
10588    if state
10589        .active
10590        .values()
10591        .any(|a| a.node == node && a.attempt == attempt)
10592    {
10593        state
10594            .active
10595            .retain(|_, a| !(a.node == node && a.attempt == attempt));
10596        if let Err(e) = state.save() {
10597            tracing::warn!("could not persist the end of a wave: {e:#}");
10598        }
10599    }
10600    // Whether or not the lease above was actually held, several worktrees
10601    // may just have built into the cache with nothing here able to name one
10602    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
10603    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
10604    // clean it might not have strictly needed; trusting a stale match would
10605    // cost it a wrong answer.
10606    if let Some(cache_dir) = cache
10607        && jobs_had_a_writer
10608    {
10609        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10610    }
10611    if let Some(guard) = cache_guard {
10612        guard.release();
10613    }
10614    collected.into_iter().flatten().collect()
10615}
10616
10617/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
10618/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
10619/// this is data collection, not the fix-specific completion contract in
10620/// [`Runner::continue_fix_report`], and applies regardless of which node
10621/// asked.
10622///
10623/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
10624/// evidence from; `Failed` does not, and correctly contributes nothing — a
10625/// timeout or crash is not itself evidence about a command the seat may have
10626/// started.
10627fn record_jobs(
10628    state: &mut RunState,
10629    node: &str,
10630    round: Option<usize>,
10631    seat: &str,
10632    out: &AgentOutcome,
10633) {
10634    let commands: &[agent::CommandEvidence] = match out {
10635        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10636        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10637    };
10638    let checked_at = Timestamp::now();
10639    for c in commands {
10640        state.jobs.push(JobRecord {
10641            node: node.to_owned(),
10642            round,
10643            seat: seat.to_owned(),
10644            id: c.id.clone(),
10645            description: c.description.clone(),
10646            checked_at,
10647            status: match c.exit_code {
10648                Some(0) => JobStatus::Completed,
10649                Some(_) => JobStatus::Failed,
10650                None => JobStatus::Unknown,
10651            },
10652            exit_code: c.exit_code,
10653            result_summary: c.result_summary.clone(),
10654            source: c.source.clone(),
10655        });
10656    }
10657}
10658
10659/// Is a review round clean, given how many reviewer seats answered against
10660/// how many the round expected?
10661///
10662/// A seat that never answered (timeout, crash, unparsable output) is not a
10663/// seat that read the patch and found nothing — treating it as such is
10664/// exactly the bug this function exists to close. Under the default `block`
10665/// policy a missing seat can never be clean; `warn` still requires the seats
10666/// that *did* answer to have found nothing blocking and verification to be
10667/// green.
10668///
10669/// `quota_missing` narrows that `block` default for exactly one cause of
10670/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
10671/// a session limit lifts by the very next round buys nothing — the seat is
10672/// asked again with the same quota — so once every missing seat is accounted
10673/// for by a quota loss (and at least one seat *did* answer, so a decision has
10674/// something to rest on) the round is decided on the panel that could answer,
10675/// same as `warn` would. A panel that lost every seat to quota is not
10676/// decided here: `answered == 0` falls through to the existing `block`
10677/// fallback so a fully collapsed panel still waits rather than landing on no
10678/// review at all.
10679fn round_is_clean(
10680    blocking: usize,
10681    e2e_ok: bool,
10682    answered: usize,
10683    expected: usize,
10684    quota_missing: usize,
10685    policy: IncompleteReviewPolicy,
10686) -> bool {
10687    if blocking != 0 || !e2e_ok {
10688        return false;
10689    }
10690    if answered == expected || policy == IncompleteReviewPolicy::Warn {
10691        return true;
10692    }
10693    answered > 0 && expected - answered <= quota_missing
10694}
10695
10696/// The review loop's own conclusion, derived entirely from its persisted
10697/// round records and the round budget that produced them — never from
10698/// `status`, so a reentry (or `gate`/`merge` reading it independently)
10699/// recomputes the identical answer regardless of what an earlier node in the
10700/// same walk, or a previous walk, did to `status`.
10701///
10702/// `None` while more rounds remain to try, including when review never ran
10703/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
10704/// gone clean, or the budget is spent, or the tree has stopped moving (see
10705/// [`STAGNANT_LIMIT`]), the answer is one of two things:
10706///
10707/// - An incomplete panel that raised nothing is missing input, not a
10708///   verified tree — never a hand-off candidate, whatever verification said
10709///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
10710/// - Otherwise, green e2e on the last round hands off (see
10711///   [`Runner::stop_reviewing`]); red e2e blocks.
10712///
10713/// A last round whose own verification is still `ResourceBlocked` — magi
10714/// itself never got a command to run, not evidence the patch is broken —
10715/// is neither: this returns `None` for it too, the same as "more rounds
10716/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
10717/// handling of that shape) instead of this cheap recomputation guessing a
10718/// verdict a real attempt never produced.
10719pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10720    if max_rounds == 0
10721        || reviews.last().is_some_and(|r| {
10722            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10723        })
10724    {
10725        return Some(RunStatus::Gating);
10726    }
10727    let last = reviews.last()?;
10728    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10729    if reviews.len() < max_rounds && !stagnant {
10730        return None;
10731    }
10732    if last.incomplete() && last.blocking == 0 {
10733        return Some(RunStatus::Blocked);
10734    }
10735    if last.e2e_status() == E2eStatus::ResourceBlocked
10736        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10737    {
10738        return None;
10739    }
10740    Some(if last.e2e.iter().all(CommandOutcome::ok) {
10741        RunStatus::Gating
10742    } else {
10743        RunStatus::Blocked
10744    })
10745}
10746
10747/// How long a re-ask may take, given the budget the first attempt had.
10748///
10749/// A `nudged` retry is a request to restate an answer the seat has already
10750/// worked out: it carries no new work, so it does not deserve the original
10751/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
10752/// 133 seconds while a third sat for over ten minutes on a resumed session
10753/// holding 410 KB of prior output - and because the retry had inherited the
10754/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
10755/// judging round whose other seats were long finished.
10756///
10757/// A quarter of the budget, with a floor so that a deliberately short timeout
10758/// does not collapse to nothing. A retry that re-sends the whole prompt
10759/// (because the seat kept no context) is the original job again, and keeps the
10760/// original budget.
10761pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10762    if nudged {
10763        (full / 4).max(Duration::from_secs(120)).min(full)
10764    } else {
10765        full
10766    }
10767}
10768
10769/// Run a wave and parse each reply, re-asking the seats whose reply was
10770/// unusable.
10771///
10772/// The re-ask is a nudge rather than the whole prompt again when the seat still
10773/// holds its conversation, which is the difference between a cheap retry and
10774/// paying for the entire candidate set twice.
10775///
10776/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
10777/// successor in `roster` is handed to it instead of being re-asked: the
10778/// handover is the retry. A seat with no successor left (a single-agent
10779/// roster, the roster's tail) is nudged as before, up to `retries` times. So
10780/// the asks to one seat in one node number at most
10781/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
10782/// is asked once (a dropped stream is nudged first), and only the last agent
10783/// of the chain gets the `retries` same-agent nudges. Each roster agent is
10784/// tried at most once per seat, walking forward from the seat's own position and never wrapping
10785/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
10786/// other failure stops the chain when the previous agent failed the same way
10787/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
10788/// [`has_context`] is false and the job's own full prompt and full budget are
10789/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
10790/// intermediate ones are not losses) and is returned as a failure like any
10791/// other absent seat — the caller decides whether the panel still has a
10792/// quorum. An empty `roster` disables handover: failures are nudged as they
10793/// always were, and a quota is simply lost. A reply that fails to parse or
10794/// validate is the prompt's doing and is only ever nudged, never handed over.
10795///
10796/// The returned [`SeatState`] names the agent that answered (or tried last).
10797#[allow(clippy::too_many_arguments)]
10798async fn ask_json_wave<T>(
10799    jobs: Vec<SeatJob>,
10800    sem: Arc<Semaphore>,
10801    retries: usize,
10802    roster: &[AgentSpec],
10803    ctx: &WaveCtx<'_>,
10804    losses: &mut Vec<QuotaLoss>,
10805    state: &mut RunState,
10806    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10807) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10808where
10809    T: serde::de::DeserializeOwned + Send + 'static,
10810{
10811    ask_wave_with(
10812        jobs,
10813        sem,
10814        retries,
10815        roster,
10816        ctx,
10817        losses,
10818        state,
10819        &|text: &str| {
10820            let v = verdict::extract_json::<T>(text)?;
10821            validate(&v)?;
10822            Ok(v)
10823        },
10824    )
10825    .await
10826}
10827
10828/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
10829/// a node whose answer is prose (deliberation) shares the same handover,
10830/// failure classification, quota bookkeeping and bounds instead of a copy.
10831///
10832/// A seat handed to another roster agent is sent the job's `handover` prompt
10833/// (when it has one) rather than `prompt`: the new agent has no session, so a
10834/// resume-style prompt would be incomplete. That holds for the handover ask
10835/// and for every nudge to that agent whose `has_context` is false.
10836#[allow(clippy::too_many_arguments)]
10837async fn ask_wave_with<T>(
10838    jobs: Vec<SeatJob>,
10839    sem: Arc<Semaphore>,
10840    retries: usize,
10841    roster: &[AgentSpec],
10842    ctx: &WaveCtx<'_>,
10843    losses: &mut Vec<QuotaLoss>,
10844    state: &mut RunState,
10845    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10846) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10847where
10848    T: Send + 'static,
10849{
10850    let n = jobs.len();
10851    let originals: Vec<SeatJob> = jobs;
10852    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10853    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10854    // Nudges each seat's *current* agent has taken — 0 for a first-ask
10855    // answer, N once it has gone through N nudges. Read back once this
10856    // returns, so a caller building a history record (`ReviewRecord`) can
10857    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
10858    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
10859    // field's own doc.
10860    let mut nudges: Vec<usize> = vec![0; n];
10861    // The agent now occupying each seat, the ids it has already been through,
10862    // where in the roster the walk began, the class of the last failure, and
10863    // the stem word of a handover not yet asked (full prompt, full budget).
10864    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10865    let mut tried: Vec<BTreeSet<String>> = specs
10866        .iter()
10867        .map(|s| BTreeSet::from([s.id.clone()]))
10868        .collect();
10869    let starts: Vec<usize> = specs
10870        .iter()
10871        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10872        .collect();
10873    let carry = ctx.carry_seats && !roster.is_empty();
10874    // Carried across rounds: ids that failed the seat earlier, and how the
10875    // last failure went (so a repeat of it is not handed over again).
10876    let carried: Vec<BTreeSet<String>> = originals
10877        .iter()
10878        .map(|j| {
10879            state
10880                .seat_history
10881                .get(&j.seat.key)
10882                .filter(|_| carry)
10883                .map(|h| h.failed.clone())
10884                .unwrap_or_default()
10885        })
10886        .collect();
10887    let mut prev: Vec<Option<FailClass>> = originals
10888        .iter()
10889        .map(|j| {
10890            state
10891                .seat_history
10892                .get(&j.seat.key)
10893                .filter(|_| carry)
10894                .and_then(|h| h.last_fail.clone())
10895        })
10896        .collect();
10897    let next_agent =
10898        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10899            let others: BTreeSet<String> = specs
10900                .iter()
10901                .enumerate()
10902                .filter(|(j, _)| *j != i)
10903                .map(|(_, s)| s.id.clone())
10904                .collect();
10905            pick_successor(
10906                roster,
10907                starts[i],
10908                tried,
10909                carry.then(|| &carried[i]),
10910                &others,
10911            )
10912            .cloned()
10913        };
10914    let mut fresh: Vec<Option<String>> = vec![None; n];
10915    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
10916    let mut pending: Vec<usize> = (0..n).collect();
10917
10918    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
10919    // asks: an agent with a successor is asked once and handed over, and only
10920    // a seat with no successor spends `retries` nudges on the same agent. This
10921    // only guarantees the loop's own termination whatever those say.
10922    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10923    for round in 0..max_rounds {
10924        if pending.is_empty() {
10925            break;
10926        }
10927        let mut batch = Vec::with_capacity(pending.len());
10928        let mut renudged: Vec<&str> = Vec::new();
10929        for &i in &pending {
10930            let src = &originals[i];
10931            // A seat now held by another agent than the job named has no
10932            // session of its own: it gets the full-context prompt whenever
10933            // it is asked in full (the handover ask, a nudge it cannot
10934            // resume).
10935            let full: &str = match &src.handover {
10936                Some(h) if specs[i].id != src.spec.id => h,
10937                _ => &src.prompt,
10938            };
10939            // The prompt and the budget are one decision: a nudge restates
10940            // finished work, a re-sent prompt redoes it.
10941            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10942                (
10943                    full.to_owned(),
10944                    src.timeout,
10945                    format!("{}-{word}-{}", src.stem, specs[i].id),
10946                )
10947            } else if nudges[i] == 0 {
10948                (full.to_owned(), src.timeout, src.stem.clone())
10949            } else {
10950                renudged.push(src.seat.key.as_str());
10951                let why = done[i]
10952                    .as_ref()
10953                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
10954                    .unwrap_or_else(|| "no parsable answer".to_owned());
10955                let nudge = prompt::nudge(&why);
10956                let nudged = has_context(&specs[i], &seats[i], src.sessions);
10957                let prompt = if nudged {
10958                    nudge
10959                } else {
10960                    format!("{full}\n\n---\n\n{nudge}")
10961                };
10962                (
10963                    prompt,
10964                    retry_budget(src.timeout, nudged),
10965                    format!("{}-retry{}", src.stem, nudges[i]),
10966                )
10967            };
10968            batch.push(SeatJob {
10969                spec: specs[i].clone(),
10970                seat: seats[i].clone(),
10971                cwd: src.cwd.clone(),
10972                prompt,
10973                timeout,
10974                allow_write: src.allow_write,
10975                sessions: src.sessions,
10976                artifacts: src.artifacts.clone(),
10977                stem,
10978                handover: None,
10979            });
10980        }
10981
10982        if !renudged.is_empty() {
10983            state.event(
10984                ctx.node,
10985                format!("retry {round}: re-asking {}", renudged.join(", ")),
10986            );
10987        }
10988        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10989        let mut still = Vec::new();
10990        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10991            seats[i] = seat;
10992            // No agent was asked: nothing to hand over, nudge, remember or
10993            // charge. The caller sees `resource_block` and stops the node.
10994            if let AgentOutcome::ResourceBlocked(why) = &out {
10995                done[i] = Some(Err(anyhow::anyhow!("{why}")));
10996                continue;
10997            }
10998            let class = FailClass::of(&out);
10999            // A dropped stream is nudged first (the conversation is still
11000            // there to pick up); only a seat whose nudges are spent hands over.
11001            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11002                && nudges[i] < retries
11003                && !roster.is_empty();
11004            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11005                let next = should_hand_over(prev[i].as_ref(), &cur)
11006                    .then(|| next_agent(i, &tried[i], &specs))
11007                    .flatten();
11008                if carry {
11009                    let h = state
11010                        .seat_history
11011                        .entry(originals[i].seat.key.clone())
11012                        .or_default();
11013                    h.failed.insert(specs[i].id.clone());
11014                    h.last_fail = Some(cur.clone());
11015                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11016                        h.last_ok = None;
11017                    }
11018                    // Saved before the next agent is asked, so a restart in
11019                    // between does not forget who failed.
11020                    if let Err(e) = state.save() {
11021                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11022                    }
11023                }
11024                if let Some(next) = next {
11025                    record_handover(
11026                        state,
11027                        ctx.node,
11028                        &originals[i].seat.key,
11029                        &specs[i].id,
11030                        &next.id,
11031                        &cur,
11032                        &fail_reason(&out),
11033                    );
11034                    tried[i].insert(next.id.clone());
11035                    prev[i] = Some(cur.clone());
11036                    seats[i] =
11037                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11038                    specs[i] = next;
11039                    fresh[i] = Some(cur.stem_word().to_owned());
11040                    nudges[i] = 0;
11041                    done[i] = Some(Err(anyhow::anyhow!(
11042                        "handed over after: {}",
11043                        fail_reason(&out)
11044                    )));
11045                    still.push(i);
11046                    continue;
11047                }
11048            }
11049            if carry
11050                && !nudge_first
11051                && let Some(cur) = class.clone()
11052            {
11053                // The chain ended here (no successor, or a repeated failure).
11054                let h = state
11055                    .seat_history
11056                    .entry(originals[i].seat.key.clone())
11057                    .or_default();
11058                h.failed.insert(specs[i].id.clone());
11059                h.last_fail = Some(cur);
11060            }
11061            let parsed = match out {
11062                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11063                AgentOutcome::Quota(o) => {
11064                    last_quota[i] = o.quota.clone();
11065                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11066                }
11067                // Not a parseable answer: the nudge loop re-asks it, which is
11068                // exactly what a dropped stream needs. Just don't hand its raw
11069                // error JSON to `extract_json`.
11070                AgentOutcome::Dropped(o) => {
11071                    let why = o
11072                        .dropped
11073                        .as_ref()
11074                        .map(|d| d.why.as_str())
11075                        .unwrap_or("the CLI ended the stream without delivering its answer");
11076                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11077                }
11078                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11079                    Err(anyhow::anyhow!(e))
11080                }
11081            };
11082            let quota = class == Some(FailClass::Quota);
11083            let failed = parsed.is_err();
11084            if !failed && matches!(ctx.node, "judge" | "vote") {
11085                // A validated answer restores this seat's quorum presence,
11086                // including on the normal path after an all-seat block.
11087                state
11088                    .quota
11089                    .retain(|loss| loss.seat != originals[i].seat.key);
11090            }
11091            done[i] = Some(parsed);
11092            if carry && !failed {
11093                let h = state
11094                    .seat_history
11095                    .entry(originals[i].seat.key.clone())
11096                    .or_default();
11097                h.failed.remove(&specs[i].id);
11098                h.last_ok = Some(specs[i].id.clone());
11099                h.last_fail = None;
11100            }
11101            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11102            // fail the same way; and never re-ask a seat that already parsed.
11103            // A failed agent that still has a successor is not re-asked
11104            // either: the handover was its remedy and has just been refused
11105            // (the chain stops on a repeated failure class). A seat with no
11106            // successor left (a single-agent roster, the roster's tail, or an
11107            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11108            let agent_failure = class.is_some()
11109                && !nudge_first
11110                && !roster.is_empty()
11111                && next_agent(i, &tried[i], &specs).is_some();
11112            if failed && !quota && !agent_failure && nudges[i] < retries {
11113                nudges[i] += 1;
11114                still.push(i);
11115            }
11116        }
11117        pending = still;
11118    }
11119
11120    // One loss per seat whose chain ended on a quota: the intermediate ones
11121    // were absorbed by a handover and are not losses.
11122    for (i, q) in last_quota.into_iter().enumerate() {
11123        if let Some(quota) = q {
11124            losses.push(QuotaLoss::from_report(
11125                originals[i].seat.key.clone(),
11126                ctx.node,
11127                Some(&quota),
11128            ));
11129        }
11130    }
11131
11132    // A stalled quorum already has its own recovery checkpoint. Refresh
11133    // its missing seats before deciding whether it can resume.
11134    if state.status != RunStatus::Stalled
11135        && ctx.node != "deliberate"
11136        && n > 0
11137        && losses.len() == n
11138        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11139    {
11140        block_machine_outage(state, ctx.node, losses);
11141        state.quota.append(losses);
11142    }
11143
11144    seats
11145        .into_iter()
11146        .zip(done)
11147        .zip(nudges)
11148        .map(|((seat, res), attempts)| {
11149            (
11150                seat,
11151                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11152                attempts,
11153            )
11154        })
11155        .collect()
11156}
11157
11158/// A node with no available agent has no task verdict. Preserve the node
11159/// checkpoint and record why a subsequent explicit resume is needed.
11160pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11161    let mut auth: Vec<_> = losses
11162        .iter()
11163        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11164        .map(|loss| loss.agent.clone())
11165        .collect();
11166    auth.sort();
11167    auth.dedup();
11168    let (kind, detail) = if auth.is_empty() {
11169        (
11170            "agent usage limit",
11171            "All seats reached their agent usage limits; resume after usage is available"
11172                .to_owned(),
11173        )
11174    } else {
11175        (
11176            "agent authentication",
11177            format!(
11178                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11179                auth.join(", ")
11180            ),
11181        )
11182    };
11183    state.resource_block = Some(crate::run::ResourceBlock {
11184        node: node.to_owned(),
11185        kind: kind.to_owned(),
11186        cause: crate::run::ResourceBlockCause::Other,
11187        detail: detail.clone(),
11188        at: Timestamp::now(),
11189    });
11190    state.event(node, detail);
11191}
11192
11193/// Did the wave end without asking any agent (see [`block_wave`])?
11194fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11195    results
11196        .iter()
11197        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11198}
11199
11200/// Settle a wave that never spawned anything: every job answers
11201/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11202/// else about the seat (session, history, quota) is touched.
11203fn block_wave(
11204    jobs: Vec<SeatJob>,
11205    state: &mut RunState,
11206    node: &str,
11207    attempt: usize,
11208    why: &str,
11209) -> Vec<(usize, SeatState, AgentOutcome)> {
11210    state.resource_wait = None;
11211    state
11212        .active
11213        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11214    if let Err(e) = state.save() {
11215        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11216    }
11217    jobs.into_iter()
11218        .enumerate()
11219        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11220        .collect()
11221}
11222
11223/// Acquire the shared build cache's lease, waiting out contention within
11224/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11225/// unbounded wait is never acceptable).
11226///
11227/// A first, non-blocking check happens before ever waiting; if it finds the
11228/// lease busy, that fact is logged as a `verify` event *and* flushed with
11229/// [`RunState::save`] immediately — not only once the wait finally succeeds
11230/// or gives up — so a `magi show` run by a different process while this one
11231/// is still waiting reads a `run.json` that says so, rather than whatever it
11232/// looked like before the wait started. The same applies to the terminal
11233/// failure: logged and saved before this returns `Err`, so a caller that
11234/// could not get the lease at all still leaves a legible record of why.
11235async fn acquire_cache_lease(
11236    state: &mut RunState,
11237    cache_dir: &Path,
11238    owner: &crate::cache::Owner,
11239    budget: Duration,
11240    node: &str,
11241    context: &str,
11242    cancel: &(dyn Fn() -> bool + Sync),
11243) -> Result<Option<crate::cache::Guard>> {
11244    let home = crate::run::home();
11245    let started = Instant::now();
11246    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11247        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11248        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11249        Err(e) => {
11250            state.event(
11251                "verify",
11252                format!("{context}: could not check the shared build cache: {e:#}"),
11253            );
11254            if let Err(e2) = state.save() {
11255                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11256            }
11257            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11258        }
11259    };
11260    state.event(
11261        "verify",
11262        format!(
11263            "{context}: waiting for the shared build cache at {} ({})",
11264            cache_dir.display(),
11265            busy.describe()
11266        ),
11267    );
11268    // Seats of this node are waiting, not running: say so, so the countdown a
11269    // reader sees only ever measures an agent that has been spawned.
11270    state.resource_wait = Some(crate::run::ResourceWait {
11271        node: context.to_owned(),
11272        kind: "shared build cache".to_owned(),
11273        holder: busy.describe(),
11274        since: Timestamp::now(),
11275        limit_secs: budget.as_secs(),
11276    });
11277    if let Err(e) = state.save() {
11278        tracing::warn!("could not persist a cache wait: {e:#}");
11279    }
11280    let remaining = budget.saturating_sub(started.elapsed());
11281    let waited = crate::cache::wait_for_unless(
11282        &home,
11283        cache_dir,
11284        owner,
11285        remaining,
11286        Duration::from_secs(5),
11287        cancel,
11288    )
11289    .await;
11290    state.resource_wait = None;
11291    let cause = match &waited {
11292        Ok(Some(_)) => None,
11293        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11294        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11295            Some(crate::run::ResourceBlockCause::Timeout {
11296                waited_secs: started.elapsed().as_secs(),
11297                limit_secs: budget.as_secs(),
11298            })
11299        }
11300        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11301    };
11302    if let Some(cause) = cause {
11303        let detail = match &waited {
11304            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11305            Err(error) => format!("{error:#}"),
11306        };
11307        state.resource_block = Some(crate::run::ResourceBlock {
11308            cause,
11309            node: node.to_owned(),
11310            kind: "shared build cache".into(),
11311            detail,
11312            at: Timestamp::now(),
11313        });
11314        state.event(
11315            "resource",
11316            state.resource_block.as_ref().unwrap().description(),
11317        );
11318    }
11319    if let Err(error) = state.save() {
11320        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11321    }
11322    waited
11323}
11324
11325/// The lease state could not be read at all (unreadable lease file, a
11326/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11327/// known to be contended, so a wave degrades to running without a shared cache
11328/// instead of stopping on a hold that [`crate::triage`] could never release.
11329#[derive(Debug)]
11330struct CacheCheckError(String);
11331
11332impl std::fmt::Display for CacheCheckError {
11333    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11334        f.write_str(&self.0)
11335    }
11336}
11337
11338impl std::error::Error for CacheCheckError {}
11339
11340/// Keep a specific acquisition outcome when the caller adds command evidence.
11341fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11342    state
11343        .resource_block
11344        .get_or_insert_with(|| crate::run::ResourceBlock {
11345            cause: crate::run::ResourceBlockCause::Other,
11346            node: node.to_owned(),
11347            kind: "shared build cache".into(),
11348            detail: format!("{error:#}"),
11349            at: Timestamp::now(),
11350        });
11351    if let Err(error) = state.save() {
11352        tracing::warn!("could not persist cache resource block: {error:#}");
11353    }
11354}
11355
11356/// Run `body` — a verify command batch — while holding the shared build
11357/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11358/// never interleave with another borrower's build against the same
11359/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11360/// timeout, or a human's own `magi review`. See the `cache` module doc for
11361/// why this matters more than Cargo's own per-target locking covers — two
11362/// *different* worktrees building the same package name/version into one
11363/// cache directory is a staleness bug, not a lock contention one.
11364///
11365/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11366/// agent waves do. Repository scheduling also has its own bounded,
11367/// cancellable resource-wait policy. Contention must not consume the
11368/// verification budget: `body` receives the full `budget` after both leases
11369/// are acquired and the cache is fresh. Without a shared cache, only
11370/// repository coordination is acquired before handing over the full budget.
11371///
11372/// A lease that cannot be acquired within the resource wait is reported as a single
11373/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11374/// verification — the same shape a spawn failure already takes in
11375/// [`run_commands`], so a caller need not special-case it.
11376#[allow(clippy::too_many_arguments)]
11377async fn with_cache_lease<'s, F, Fut>(
11378    state: &'s mut RunState,
11379    cache_dir: Option<&Path>,
11380    node: &str,
11381    seat: &str,
11382    worktree: &Path,
11383    head: &str,
11384    budget: Duration,
11385    context: &str,
11386    wait: &crate::repo_lock::WaitPolicy,
11387    body: F,
11388) -> Result<(Vec<CommandOutcome>, bool)>
11389where
11390    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11391    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11392{
11393    // Match editing waves: cache first, repository second. In particular,
11394    // pre_gate must never hold the repository while waiting for a wave's cache.
11395    let started = Instant::now();
11396    let home = crate::run::home();
11397    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11398    let guard = if let Some(cache_dir) = cache_dir {
11399        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11400        match acquire_cache_lease(
11401            state,
11402            cache_dir,
11403            &owner,
11404            resource_limit,
11405            node,
11406            context,
11407            &|| wait.cancelled(),
11408        )
11409        .await
11410        .and_then(|g| {
11411            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11412        }) {
11413            Ok(g) => Some(g),
11414            Err(e) => {
11415                record_cache_error(state, node, &e);
11416                return Ok((
11417                    vec![CommandOutcome {
11418                        command: "(waiting for the shared build cache)".into(),
11419                        code: None,
11420                        output_tail: e.to_string(),
11421                        duration_ms: started.elapsed().as_millis() as u64,
11422                        resource_blocked: true,
11423                    }],
11424                    false,
11425                ));
11426            }
11427        }
11428    } else {
11429        None
11430    };
11431    let _coordination = coordinate_operation(state, worktree, wait).await?;
11432    if let Some(cache_dir) = cache_dir {
11433        let identity = crate::cache::Identity::new(worktree, head);
11434        if let Err(e) = crate::cache::ensure_fresh(
11435            &home,
11436            cache_dir,
11437            &identity,
11438            &state.config.verify.cargo_manifests(),
11439        ) {
11440            record_cache_error(state, node, &e);
11441            state.event(
11442                "verify",
11443                format!(
11444                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11445                    worktree.display(),
11446                    short(head)
11447                ),
11448            );
11449            if let Some(guard) = guard {
11450                guard.release();
11451            }
11452            return Ok((
11453                vec![CommandOutcome {
11454                    command: "(confirming the shared build cache is fresh)".into(),
11455                    code: None,
11456                    output_tail: e.to_string(),
11457                    duration_ms: started.elapsed().as_millis() as u64,
11458                    resource_blocked: true,
11459                }],
11460                false,
11461            ));
11462        }
11463    }
11464    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11465    if !timed_out_pids.is_empty() {
11466        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11467    }
11468    // Drop repository coordination before releasing the cache, reversing the
11469    // acquisition order. A cancelled/error future also drops both guards.
11470    drop(_coordination);
11471    if let Some(guard) = guard {
11472        guard.release();
11473    }
11474    Ok((outcomes, retried))
11475}
11476
11477/// Poll `pids` — commands [`run_commands`] reports as still running when its
11478/// own timeout elapsed — until every one is confirmed gone, or
11479/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11480///
11481/// Real confirmation where confirmation is possible, not a substitute for
11482/// full process-tree observation: a grandchild the timed-out process spawned
11483/// and that survives independently of it is invisible to a pid check the
11484/// same way it always was, and continuing to observe and collect *that*
11485/// stays a different piece of work with its own owner. This only narrows a
11486/// fixed blind wait into an actual check of the pids this process does know
11487/// about.
11488async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11489    wait_for_pids_with(
11490        pids,
11491        crate::proc::pid_alive,
11492        LEASE_RELEASE_POLL,
11493        LEASE_RELEASE_MAX_WAIT,
11494    )
11495    .await;
11496}
11497
11498/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
11499/// interval and ceiling supplied by the caller, so the polling *logic* -
11500/// returns as soon as every pid reports dead, gives up at the ceiling
11501/// otherwise - is testable on millisecond durations without asking the real
11502/// OS about a pid at all.
11503async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11504    pids: &[u32],
11505    alive: F,
11506    poll: Duration,
11507    max_wait: Duration,
11508) {
11509    let deadline = Instant::now() + max_wait;
11510    loop {
11511        if pids.iter().all(|&pid| !alive(pid)) {
11512            return;
11513        }
11514        if Instant::now() >= deadline {
11515            return;
11516        }
11517        tokio::time::sleep(poll).await;
11518    }
11519}
11520
11521/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
11522/// admission that it could not even get a verify command to run, as opposed
11523/// to evidence the command actually produced? A caller that would otherwise
11524/// read a resource-blocked outcome as a red command must check this first:
11525/// see [`Runner::gate`], which retries rather than records `Blocked` when
11526/// this is true.
11527fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11528    outcomes.iter().any(|o| o.resource_blocked)
11529}
11530
11531/// What [`Runner::gate_fix_round`] decided.
11532pub(crate) enum GateFix {
11533    /// The tree changed and `verify.e2e` is still green: run the gate again.
11534    Retry,
11535    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
11536    /// last failure stands and the run ends blocked.
11537    Stop,
11538    /// `verify.e2e` could not run after the fix (magi's own contention):
11539    /// decide nothing now, a later reentry retries.
11540    Defer,
11541}
11542
11543/// Is every red command in `outcomes` an ordinary failure the code could
11544/// explain: it ran, exited non-zero, and said something?
11545///
11546/// A timeout, a spawn failure and a killed process all leave `code` `None`;
11547/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
11548/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
11549/// what the command is stays the gate's business.
11550pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11551    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11552    red.peek().is_some()
11553        && red.all(|o| {
11554            !o.resource_blocked
11555                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11556                && !o.output_tail.trim().is_empty()
11557        })
11558}
11559
11560/// Describe one verify command's outcome for the event log, distinguishing a
11561/// build/link failure — the toolchain never produced a binary to run — from
11562/// an actual test failure, since only the latter is a verdict on the patch.
11563fn e2e_outcome_label(o: &CommandOutcome) -> String {
11564    if o.ok() {
11565        return "pass".to_owned();
11566    }
11567    let reason = if o.build_failed() {
11568        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11569    } else {
11570        format!("FAIL ({:?})", o.code)
11571    };
11572    format!(
11573        "{reason}\n{}",
11574        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11575    )
11576}
11577
11578/// Run `verify.e2e`, retrying once if the first attempt could not build or
11579/// link — a build/link failure is frequently a race against a shared
11580/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
11581/// `verify` event per command, tagged with `context` (normally `"round N"`)
11582/// so the two call sites that need this — the ordinary per-round leg in
11583/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
11584/// it will ever call a round green — read identically in the event log.
11585async fn run_e2e_with_retry(
11586    state: &mut RunState,
11587    shell: &[String],
11588    commands: &[String],
11589    worktree: &Path,
11590    timeout: Duration,
11591    context: &str,
11592) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11593    let (mut e2e, mut timed_out_pids) = run_commands(
11594        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11595    )
11596    .await;
11597    for o in &e2e {
11598        state.event(
11599            "verify",
11600            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11601        );
11602    }
11603    // A build/link failure is not a verdict on the patch — it is frequently a
11604    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
11605    // one retry before letting a red like that decide the round.
11606    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11607    if verify_retried {
11608        state.event(
11609            "verify",
11610            format!(
11611                "{context}: verify could not build/link, not a test result — retrying once \
11612                 before concluding"
11613            ),
11614        );
11615        let retried = run_commands(
11616            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11617        )
11618        .await;
11619        e2e = retried.0;
11620        // Both attempts' timeouts matter, not just the last one: the first
11621        // attempt's descendants may still be alive alongside the retry's.
11622        timed_out_pids.extend(retried.1);
11623        for o in &e2e {
11624            state.event(
11625                "verify",
11626                format!(
11627                    "{context}: retry `{}` -> {}",
11628                    o.command,
11629                    e2e_outcome_label(o)
11630                ),
11631            );
11632        }
11633    }
11634    (e2e, verify_retried, timed_out_pids)
11635}
11636
11637/// Run configured shell commands in `cwd`, in order. The second element is
11638/// the pid of every command that hit `timeout` and was still running when
11639/// this stopped waiting on it (best-effort: `None` when the platform did not
11640/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
11641/// that releases a shared resource afterward needs to know.
11642///
11643/// Records `task` into [`RunState::active`] at every command boundary
11644/// (`RunState::task_command`) and clears it once the whole list has run
11645/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
11646/// for minutes with no seat and no output of its own to show for it (see
11647/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
11648/// apart from "not yet run" without this), and this is the only place that
11649/// knows which command is running right now and how many are left. Three
11650/// saves per command — start, not per second — matching the same "only at a
11651/// boundary" rule [`wave`] already follows for seats.
11652#[allow(clippy::too_many_arguments)]
11653async fn run_commands(
11654    state: &mut RunState,
11655    node: &str,
11656    task: &str,
11657    attempt: usize,
11658    shell: &[String],
11659    commands: &[String],
11660    cwd: &Path,
11661    timeout: Duration,
11662) -> (Vec<CommandOutcome>, Vec<u32>) {
11663    if commands.is_empty() {
11664        // Nothing to mark as running and nothing to clear — an empty list
11665        // means "not configured", and touching `active` (or the disk) over
11666        // that would be a write for every round of a repo with no
11667        // `verify.e2e` / `verify.gate` commands at all.
11668        return (Vec::new(), Vec::new());
11669    }
11670    let mut out = Vec::new();
11671    let mut timed_out_pids = Vec::new();
11672    let total = commands.len();
11673    for (idx, command) in commands.iter().enumerate() {
11674        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11675        if let Err(e) = state.save() {
11676            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11677        }
11678        let started = Instant::now();
11679        let mut cmd = tokio::process::Command::new(&shell[0]);
11680        cmd.quiet();
11681        // Verify the checkout as a standalone caller. Worker ownership and
11682        // pinned executable selection belong to this driver, not to tests or
11683        // tools it runs. Preserve home/seat context and Running's containment.
11684        for key in crate::worker::INVOCATION_ENV {
11685            cmd.env_remove(key);
11686        }
11687        cmd.args(&shell[1..])
11688            .arg(command)
11689            .current_dir(cwd)
11690            .stdin(std::process::Stdio::null())
11691            .stdout(std::process::Stdio::piped())
11692            .stderr(std::process::Stdio::piped())
11693            .kill_on_drop(true);
11694        let mut outputs = Vec::new();
11695        let protected = (|| -> Result<()> {
11696            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
11697                outputs.push(guard);
11698            }
11699            if let Some(cache) = state.config.cache_dir() {
11700                outputs.push(crate::retention::OutputLease::cache(
11701                    &crate::run::home(),
11702                    &cache,
11703                    false,
11704                )?);
11705            }
11706            Ok(())
11707        })();
11708        if let Err(e) = protected {
11709            out.push(CommandOutcome {
11710                command: command.clone(),
11711                code: None,
11712                output_tail: format!("output ownership unavailable: {e:#}"),
11713                duration_ms: 0,
11714                resource_blocked: true,
11715            });
11716            continue;
11717        }
11718        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
11719        let (code, body) = match spawned {
11720            Ok(mut running) => {
11721                let pid = running.child().id();
11722                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
11723                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
11724                let mut paths = vec![cwd.to_owned()];
11725                if let Some(cache) = state.config.cache_dir() {
11726                    paths.push(cache);
11727                }
11728                let mut pressure = None;
11729                let status = tokio::select! {
11730                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
11731                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
11732                        pressure = Some(reason); None
11733                    }
11734                };
11735                let (code, failure) = match status {
11736                    Some(Ok(Ok(status))) => (
11737                        status.code(),
11738                        status
11739                            .code()
11740                            .is_none()
11741                            .then(|| format!("command terminated: {status}")),
11742                    ),
11743                    Some(Ok(Err(error))) => {
11744                        (None, Some(format!("failed waiting for command: {error}")))
11745                    }
11746                    Some(Err(_)) => {
11747                        // Leased cleanup confirms process-tree death itself.
11748                        // Unleased commands still need the caller's PID check
11749                        // after kill-on-drop, before releasing cache ownership.
11750                        if !running.coordinated()
11751                            && let Some(pid) = pid
11752                        {
11753                            timed_out_pids.push(pid);
11754                        }
11755                        (
11756                            None,
11757                            Some(format!("timed out after {}s", timeout.as_secs())),
11758                        )
11759                    }
11760                    None => (None, None),
11761                };
11762                running.finish().await;
11763                if let Some(reason) = pressure {
11764                    state.resource_block = Some(crate::run::ResourceBlock {
11765                        cause: crate::run::ResourceBlockCause::Other,
11766                        node: node.to_owned(),
11767                        kind: "disk pressure".to_owned(),
11768                        detail: reason.clone(),
11769                        at: Timestamp::now(),
11770                    });
11771                    out.push(CommandOutcome {
11772                        command: command.clone(),
11773                        code: None,
11774                        output_tail: reason,
11775                        duration_ms: started.elapsed().as_millis() as u64,
11776                        resource_blocked: true,
11777                    });
11778                    break;
11779                }
11780                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11781                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11782                if let Some(failure) = failure {
11783                    body.push('\n');
11784                    body.push_str(&failure);
11785                }
11786                (code, body)
11787            }
11788            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11789        };
11790        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
11791        if !body.is_empty() {
11792            // The full text is kept whatever the summary dropped. A fresh
11793            // name per command so a resume or retry never overwrites an
11794            // earlier run's output. A failed write changes no verdict.
11795            match crate::run::write_verification_artifact(state, task, &body) {
11796                Ok(name) if summary != body => {
11797                    summary = format!("[full output: artifacts/{name}]\n{summary}");
11798                }
11799                Ok(_) => {}
11800                Err(e) => {
11801                    let warning = format!("could not save the full {task} output: {e:#}");
11802                    tracing::warn!("{warning}");
11803                    state.event(node, warning);
11804                    if summary != body {
11805                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
11806                    }
11807                }
11808            }
11809        }
11810        out.push(CommandOutcome {
11811            command: command.clone(),
11812            code,
11813            output_tail: summary,
11814            duration_ms: started.elapsed().as_millis() as u64,
11815            resource_blocked: false,
11816        });
11817    }
11818    state.task_finished(task);
11819    if let Err(e) = state.save() {
11820        tracing::warn!("could not persist the end of {task}: {e:#}");
11821    }
11822    (out, timed_out_pids)
11823}
11824
11825/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
11826/// section (`report::run`) and in the `merge` event this node records — for
11827/// the operator to run by hand.
11828///
11829/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
11830/// branch whose ruleset forbids merge commits (GitHub's "must not contain
11831/// merge commits", or "require linear history") rejects the push a `--no-ff`
11832/// merge would produce, which is exactly the guidance this function replaces.
11833/// `message`'s first line becomes the squash commit's subject, matching the
11834/// note `report::run` prints alongside this command — see that function for
11835/// why an explicit subject is not optional there.
11836fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11837    let repo = repo.display();
11838    match style {
11839        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11840        MergeStyle::Squash => {
11841            // The subject sits inside double quotes, and a title an agent
11842            // wrote may carry the characters that break out of them.
11843            let subject = message
11844                .lines()
11845                .next()
11846                .unwrap_or(branch)
11847                .replace(['\\', '"', '$', '`'], "");
11848            format!(
11849                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11850            )
11851        }
11852        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11853    }
11854}
11855
11856/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
11857/// under the hood, rejects a `title` over 256 characters and the whole
11858/// command fails — no PR at all, for a run whose body was otherwise fine
11859/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
11860/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
11861/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
11862/// this bound. It is a margin, not a guarantee — a title packed
11863/// with multi-unit characters could still in principle land close to the
11864/// edge, but a real task title's occasional emoji or accented letter fits
11865/// comfortably inside it.
11866const PR_TITLE_MAX: usize = 240;
11867
11868/// A pull request title from the opening line of `text`, or `None` when that
11869/// line is not English (GitHub text is) or has no letters.
11870///
11871/// A line within `max` is kept as is. A longer one is cut at the end of its
11872/// first sentence when that falls inside `max`, else at a word boundary with a
11873/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
11874/// make every merely-truncated title look non-English). The language check
11875/// runs on the kept text before any mark is added, so only what GitHub will
11876/// show is judged: an English opening followed by non-ASCII far past the cut
11877/// still passes.
11878fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11879    let line = queue::first_line(text)?;
11880    let chars: Vec<char> = line.chars().collect();
11881    let (kept, mark) = if chars.len() <= max {
11882        (line.to_owned(), "")
11883    } else if let Some(end) = sentence_end(&chars, max) {
11884        (chars[..end].iter().collect::<String>(), "")
11885    } else {
11886        let room = max.saturating_sub(3);
11887        // Cut at the last space inside the room; when the char just past the
11888        // room is a space the room already ends on a word.
11889        let cut = if chars[room].is_whitespace() {
11890            room
11891        } else {
11892            chars[..room]
11893                .iter()
11894                .rposition(|c| c.is_whitespace())
11895                .unwrap_or(room)
11896        };
11897        let head: String = chars[..cut].iter().collect();
11898        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11899        (head.to_owned(), "...")
11900    };
11901    if kept.is_empty() {
11902        return None;
11903    }
11904    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11905        return None;
11906    }
11907    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11908        return None;
11909    }
11910    Some(format!("{kept}{mark}"))
11911}
11912
11913/// The char length of the first sentence of `chars` when it ends within `max`
11914/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
11915/// abbreviations so `e.g. foo` does not end a title early.
11916fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11917    const MIN: usize = 20;
11918    for i in MIN..max.min(chars.len()) {
11919        if !matches!(chars[i], '.' | '!' | '?') {
11920            continue;
11921        }
11922        let Some(&next) = chars.get(i + 1) else {
11923            continue;
11924        };
11925        if !next.is_whitespace() {
11926            continue;
11927        }
11928        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11929        if after.is_some_and(|c| c.is_ascii_lowercase()) {
11930            continue;
11931        }
11932        let word: String = chars[..i]
11933            .iter()
11934            .rev()
11935            .take_while(|c| !c.is_whitespace())
11936            .collect::<Vec<_>>()
11937            .into_iter()
11938            .rev()
11939            .collect();
11940        let word = word.to_ascii_lowercase();
11941        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11942            continue;
11943        }
11944        let end = chars[..i]
11945            .iter()
11946            .rposition(|c| !c.is_whitespace())
11947            .map_or(i, |p| p + 1);
11948        return Some(end);
11949    }
11950    None
11951}
11952
11953/// What `merge = "pr"` (and the merge commit of the other modes) says about a
11954/// change: a title and a body describing what was *implemented*, not the task
11955/// that asked for it. A task reads as a request; a reader of the merged
11956/// history wants the change.
11957struct PrMessage {
11958    title: String,
11959    body: String,
11960}
11961
11962impl PrMessage {
11963    /// Title, blank line, body. The first line is the squash/merge commit
11964    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
11965    /// has to stay one sensible line.
11966    fn commit_message(&self) -> String {
11967        format!("{}\n\n{}", self.title, self.body)
11968    }
11969}
11970
11971/// The text after a leading `TITLE:` (any case) on `line`.
11972fn title_marker(line: &str) -> Option<&str> {
11973    let line = line.trim();
11974    let head = line.get(..6)?;
11975    head.eq_ignore_ascii_case("title:")
11976        .then(|| line[6..].trim())
11977}
11978
11979/// The implementer's own one-line title: the `TITLE:` line the implement
11980/// prompt asks for at the top of its SUMMARY. Candidate commits are all
11981/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
11982/// source, and a title that says as much is refused here too.
11983fn summary_title(summary: &str) -> Option<String> {
11984    let first = summary.lines().find(|l| !l.trim().is_empty())?;
11985    let raw = title_marker(first)?;
11986    if raw.is_empty() {
11987        return None;
11988    }
11989    let title = queue::title_from(raw, PR_TITLE_MAX);
11990    let lower = title.to_ascii_lowercase();
11991    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11992        return None;
11993    }
11994    Some(title)
11995}
11996
11997/// How `open_review`'s instruction begins; see [`landing_title`].
11998const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11999
12000/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12001pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12002
12003/// Does this run review work that already existed, rather than implement a
12004/// task? Runs recorded before `reviewed_commits` existed carry only the
12005/// candidate marker.
12006pub(crate) fn is_review_run(state: &RunState) -> bool {
12007    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12008}
12009
12010/// The title of a review-only run: the subject of the oldest commit under
12011/// review. Later commits are usually fixups, and `instruction` is the review
12012/// prompt, which says nothing about the change. GitHub text is English, so a
12013/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12014fn review_title(state: &RunState) -> Option<String> {
12015    english_subject(
12016        state.reviewed_commits.as_ref()?.first()?,
12017        state.config.graph.github_english(),
12018    )
12019}
12020
12021/// `raw` as a pull request title, or `None` when it is blank, not English
12022/// (GitHub text is), or one of magi's own candidate commit subjects.
12023fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12024    let raw = raw.trim();
12025    if raw.is_empty() {
12026        return None;
12027    }
12028    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12029        return None;
12030    }
12031    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12032        return None;
12033    }
12034    let title = queue::title_from(raw, PR_TITLE_MAX);
12035    let lower = title.to_ascii_lowercase();
12036    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12037        return None;
12038    }
12039    Some(title)
12040}
12041
12042/// What a review-only run's branch says about itself, read at the moment the
12043/// pull request is opened.
12044#[derive(Debug, Clone, PartialEq, Eq)]
12045struct BranchFacts {
12046    /// `(subject, body)` of each commit, oldest first.
12047    commits: Vec<(String, String)>,
12048    /// Trimmed `git diff --stat`.
12049    stat: String,
12050}
12051
12052/// Longest diff stat shown: this many file lines plus the summary line.
12053const STAT_FILE_LINES: usize = 25;
12054/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12055const COMMITS_MAX_CHARS: usize = 20_000;
12056
12057/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12058/// say or finds nothing, so the caller falls back to what the run recorded.
12059async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12060    let commits = git::commit_log(repo, base, branch).await.ok()?;
12061    if commits.is_empty() {
12062        return None;
12063    }
12064    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12065    let lines: Vec<&str> = stat.lines().collect();
12066    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12067        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12068        let more = format!(" ... {omitted} more file(s)");
12069        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12070        kept.push(&more);
12071        kept.push(lines[lines.len() - 1]);
12072        kept.join("\n")
12073    } else {
12074        lines.join("\n")
12075    };
12076    Some(BranchFacts { commits, stat })
12077}
12078
12079/// Defang what would break the surrounding markdown: a closing `</details>`
12080/// and a code fence.
12081fn markdown_safe(text: &str) -> String {
12082    text.replace("</details>", "&lt;/details&gt;")
12083        .replace("\x60\x60\x60", "~~~")
12084}
12085
12086fn neutral_title(state: &RunState, winner: char) -> String {
12087    format!(
12088        "chore: land candidate {} of run {}",
12089        winner.to_ascii_uppercase(),
12090        state.id
12091    )
12092}
12093
12094/// Reuse the graph's bounded fixer without starting another graph or driver.
12095/// Copy the working state back even on errors so spent rounds remain visible.
12096pub(crate) async fn repair_landing_gate(
12097    state: &mut RunState,
12098    winner: &Candidate,
12099    outcomes: &[CommandOutcome],
12100) -> Result<GateFix> {
12101    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12102        return Ok(GateFix::Stop);
12103    }
12104    let mut runner = Runner {
12105        roles: state.config.resolve_roles()?,
12106        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12107        state: state.clone(),
12108        pause: Pause::new(),
12109        interrupt: Pause::new(),
12110    };
12111    let result = runner.gate_fix_round(winner, outcomes).await;
12112    *state = runner.state;
12113    result
12114}
12115
12116/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12117/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12118pub(crate) async fn verify_current_head(
12119    state: &mut RunState,
12120    winner: &Candidate,
12121    wait: &crate::repo_lock::WaitPolicy,
12122) -> Result<bool> {
12123    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12124    // Preserve the previous record before replacing it, including on crash recovery.
12125    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12126        state
12127            .verification_history
12128            .push(crate::run::VerificationEvidence {
12129                superseded_by: Some(head.clone()),
12130                archived_at: Some(Timestamp::now()),
12131                fixes_spent: Some(state.gate_fixes.len()),
12132                gate_head: None,
12133                gate_ran: false,
12134                gate: Vec::new(),
12135                e2e: state.head_e2e.take(),
12136            });
12137    }
12138    if let Some(e) = &state.head_e2e
12139        && e.head == head
12140    {
12141        if e.passed {
12142            return Ok(true);
12143        }
12144        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12145            state.status = RunStatus::Blocked;
12146            state.save()?;
12147            return Ok(false);
12148        }
12149    } else if state.reviews.last().is_some_and(|r| {
12150        r.verified_head.as_deref() == Some(head.as_str())
12151            && (r.e2e_status() == E2eStatus::Passed
12152                || (r.e2e_status() == E2eStatus::NotConfigured
12153                    && state.config.verify.e2e.is_empty()))
12154    }) || state
12155        .gate_repair_e2e
12156        .as_ref()
12157        .is_some_and(|e| e.head == head && e.passed)
12158    {
12159        return Ok(true);
12160    }
12161    if let Some(e) = &state.gate_repair_e2e
12162        && e.head == head
12163        && !e.passed
12164        && !e.outcomes.is_empty()
12165        && !verify_inconclusive(&e.outcomes)
12166    {
12167        state.status = RunStatus::Blocked;
12168        state.save()?;
12169        return Ok(false);
12170    }
12171    state.head_e2e = Some(crate::run::GateRepairE2e {
12172        head: head.clone(),
12173        passed: false,
12174        outcomes: Vec::new(),
12175    });
12176    state.save()?;
12177    let commands = state.config.verify.e2e.clone();
12178    let shell = state.config.shell();
12179    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12180    let cache_dir = state.config.cache_dir();
12181    let outcomes = if commands.is_empty() {
12182        Vec::new()
12183    } else {
12184        let (outcomes, _) = with_cache_lease(
12185            state,
12186            cache_dir.as_deref(),
12187            "e2e",
12188            "e2e",
12189            &winner.worktree,
12190            &head,
12191            timeout,
12192            "current head e2e",
12193            wait,
12194            |state, budget| async move {
12195                run_e2e_with_retry(
12196                    state,
12197                    &shell,
12198                    &commands,
12199                    &winner.worktree,
12200                    budget,
12201                    "current head e2e",
12202                )
12203                .await
12204            },
12205        )
12206        .await?;
12207        outcomes
12208    };
12209    let unresolved = verify_inconclusive(&outcomes);
12210    let passed = !unresolved
12211        && outcomes.iter().all(CommandOutcome::ok)
12212        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12213    state.head_e2e = Some(crate::run::GateRepairE2e {
12214        head: head.clone(),
12215        passed,
12216        outcomes,
12217    });
12218    if !passed && !unresolved {
12219        state.status = RunStatus::Blocked;
12220        state.event(
12221            "e2e",
12222            format!("verify.e2e failed on head {head}; nothing published"),
12223        );
12224    }
12225    state.save()?;
12226    Ok(passed)
12227}
12228
12229/// Recheck a repair's e2e requirement across failures, resource deferrals and
12230/// process interruptions. A successful result only applies to its exact SHA.
12231pub(crate) async fn check_gate_repair_e2e(
12232    state: &mut RunState,
12233    winner: &Candidate,
12234    wait: &crate::repo_lock::WaitPolicy,
12235) -> Result<GateFix> {
12236    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12237        return Ok(GateFix::Retry);
12238    }
12239    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12240    if state
12241        .gate_repair_e2e
12242        .as_ref()
12243        .is_some_and(|r| r.head == head && r.passed)
12244    {
12245        return Ok(GateFix::Retry);
12246    }
12247    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12248        head: head.clone(),
12249        passed: false,
12250        outcomes: Vec::new(),
12251    });
12252    state.save()?;
12253    let commands = state.config.verify.e2e.clone();
12254    let outcomes = if commands.is_empty() {
12255        Vec::new()
12256    } else {
12257        let shell = state.config.shell();
12258        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12259        let cache_dir = state.config.cache_dir();
12260        let (outcomes, _) = with_cache_lease(
12261            state,
12262            cache_dir.as_deref(),
12263            "e2e",
12264            "e2e",
12265            &winner.worktree,
12266            &head,
12267            timeout,
12268            "gate repair e2e",
12269            wait,
12270            |state, budget| async move {
12271                run_e2e_with_retry(
12272                    state,
12273                    &shell,
12274                    &commands,
12275                    &winner.worktree,
12276                    budget,
12277                    "gate repair e2e",
12278                )
12279                .await
12280            },
12281        )
12282        .await?;
12283        outcomes
12284    };
12285    let deferred = verify_inconclusive(&outcomes);
12286    let passed = outcomes.iter().all(CommandOutcome::ok)
12287        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12288    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12289        head,
12290        passed,
12291        outcomes,
12292    });
12293    state.save()?;
12294    if deferred {
12295        Ok(GateFix::Defer)
12296    } else if !passed {
12297        state.event(
12298            "gate",
12299            "verify.e2e failed after gate repair; nothing published",
12300        );
12301        Ok(GateFix::Stop)
12302    } else {
12303        Ok(GateFix::Retry)
12304    }
12305}
12306
12307/// Run the same configured gate for a landing head and a fresh graph.
12308pub(crate) async fn run_landing_gate(
12309    state: &mut RunState,
12310    winner: &Candidate,
12311    wait: &crate::repo_lock::WaitPolicy,
12312) -> Result<Vec<CommandOutcome>> {
12313    let shell = state.config.shell();
12314    let gate_commands = state.config.verify.gate.clone();
12315    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12316    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12317        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12318        Vec::new()
12319    } else {
12320        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12321        let cache_dir = state.config.cache_dir();
12322        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12323        let (outcomes, _) = with_cache_lease(
12324            state,
12325            cache_dir.as_deref(),
12326            "gate",
12327            "gate",
12328            &winner.worktree,
12329            &head,
12330            timeout,
12331            "final gate",
12332            wait,
12333            |state, budget| {
12334                let shell = shell.clone();
12335                let gate_commands = gate_commands.clone();
12336                let winner = winner.clone();
12337                async move {
12338                    Runner::run_pre_gate(state, &winner).await;
12339                    // Bind command evidence after mechanical fixes, before commands start.
12340                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12341                    let (outcomes, timed_out_pids) = run_commands(
12342                        state,
12343                        "gate",
12344                        "gate",
12345                        0,
12346                        &shell,
12347                        &gate_commands,
12348                        &winner.worktree,
12349                        budget,
12350                    )
12351                    .await;
12352                    (outcomes, false, timed_out_pids)
12353                }
12354            },
12355        )
12356        .await?;
12357        outcomes
12358    };
12359    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12360    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12361        let mut outcomes = outcomes;
12362        outcomes.push(CommandOutcome {
12363            command: "gate head stability".into(), code: None,
12364            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12365            duration_ms: 0, resource_blocked: false,
12366        });
12367        return Ok(outcomes);
12368    }
12369    if outcomes.is_empty() {
12370        // Nothing configured to check — distinct from every other
12371        // silence in this run's event log, since an empty `gate` alone
12372        // no longer says whether the gate ran at all (see
12373        // `RunState::gate_ran`'s own doc).
12374        state.event("gate", "no gate commands configured; not tested");
12375    }
12376    for o in &outcomes {
12377        state.event(
12378            "gate",
12379            format!(
12380                "`{}` -> {}",
12381                o.command,
12382                if o.ok() {
12383                    "pass".to_owned()
12384                } else {
12385                    format!(
12386                        "FAIL ({:?})\n{}",
12387                        o.code,
12388                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12389                    )
12390                }
12391            ),
12392        );
12393    }
12394    Ok(outcomes)
12395}
12396
12397/// The pull request title to hand to `land::merge_subject`. A review-only run
12398/// opened by an earlier build titled its pull request with the review prompt;
12399/// that title is dropped (empty, so the fallback applies) rather than landed.
12400/// Any other title, including an operator's rename, passes through untouched,
12401/// and so does every title of a run that implements a task.
12402pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12403    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12404        ""
12405    } else {
12406        pr_title
12407    }
12408}
12409
12410/// What the squash subject falls back to when the pull request title is empty
12411/// or candidate-shaped: for a review-only run the derived title, never the
12412/// review prompt held in `instruction`.
12413pub fn landing_subject_source(state: &RunState) -> String {
12414    if is_review_run(state) {
12415        let winner = state.candidates.first().map_or('A', |c| c.label);
12416        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12417    }
12418    state.instruction.clone()
12419}
12420
12421/// `summary` without its `TITLE:` line, which the pull request title already
12422/// carries.
12423fn summary_without_title(summary: &str) -> String {
12424    let mut lines = summary.trim().lines().peekable();
12425    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12426        lines.next();
12427    }
12428    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12429}
12430
12431/// The pull request title and body for the winning candidate.
12432///
12433/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12434/// the task's own opening line via [`queue::title_from`] when there is none.
12435/// `state.instruction` can open with blank lines (`task_text` only rejects a
12436/// body that is blank *entirely*), which `title_from` skips.
12437///
12438/// Publishing uses an independent description generated from the final diff
12439/// and structured evidence. Internal summaries and reports remain local.
12440/// Non-publishing previews may display the original task and candidate summary.
12441/// The footer retains the run and candidate provenance.
12442#[cfg(test)]
12443fn pr_message(state: &RunState, winner: char) -> PrMessage {
12444    pr_message_with(state, winner, None)
12445}
12446
12447/// [`pr_message`] with what the branch of a review-only run says about itself.
12448/// `facts` is ignored for a run that implements a task.
12449#[cfg(test)]
12450fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12451    let raw = pr_message_raw(
12452        state,
12453        winner,
12454        facts,
12455        crate::pr_outcome::FinalHead::Unreported,
12456        ChangeNote::NotAsked,
12457    );
12458    let id = crate::scrub::Identity::current();
12459    PrMessage {
12460        title: crate::scrub::scrub(&raw.title, &id),
12461        body: crate::scrub::scrub(&raw.body, &id),
12462    }
12463}
12464
12465/// What the public description may say about the change itself.
12466#[derive(Debug, Clone, Copy)]
12467enum ChangeNote<'a> {
12468    /// A local preview or non-publishing merge mode: no public generation needed.
12469    NotAsked,
12470    /// A summary written from the final diff.
12471    Summary(&'a crate::run::ChangeSummary),
12472    /// A summary was asked for and could not be produced; the text says why.
12473    Unavailable(&'a str),
12474}
12475
12476fn pr_message_raw(
12477    state: &RunState,
12478    winner: char,
12479    facts: Option<&BranchFacts>,
12480    head: crate::pr_outcome::FinalHead<'_>,
12481    change: ChangeNote<'_>,
12482) -> PrMessage {
12483    if !matches!(change, ChangeNote::NotAsked) {
12484        let (title, mut body) = match change {
12485            ChangeNote::Summary(c) => (
12486                c.title.clone(),
12487                format!("## Summary\n\n{}\n\n", c.body.trim()),
12488            ),
12489            ChangeNote::Unavailable(why) => (
12490                neutral_title(state, winner),
12491                format!(
12492                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12493                ),
12494            ),
12495            ChangeNote::NotAsked => unreachable!(),
12496        };
12497        body.push_str(&public_pr_evidence(state, head));
12498        body.push_str(&format!(
12499            "\n---\nmagi:run/{} magi:candidate-{}\n",
12500            state.id,
12501            winner.to_ascii_lowercase()
12502        ));
12503        return PrMessage { title, body };
12504    }
12505    pr_preview(state, winner, facts, head)
12506}
12507
12508/// Append public facts from the run records after every prose generation,
12509/// including posting-gate rewrites. The agent cannot drop this provenance.
12510fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12511    let mut body = crate::pr_outcome::review_rounds(state);
12512    body.push_str(&crate::pr_outcome::final_verification(state, head));
12513    let open = state.last_round_findings();
12514    if !open.is_empty() {
12515        body.push_str("## Open review findings\n\n");
12516        for f in open {
12517            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12518        }
12519        body.push('\n');
12520    }
12521    if let Some(c) = state.change_summary.as_ref()
12522        && head == crate::pr_outcome::FinalHead::At(&c.head)
12523    {
12524        body.push_str(&format!(
12525            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12526            c.base, c.head
12527        ));
12528    }
12529    body
12530}
12531
12532/// Local-only preview of the operator's task and candidate summary.
12533fn pr_preview(
12534    state: &RunState,
12535    winner: char,
12536    facts: Option<&BranchFacts>,
12537    head: crate::pr_outcome::FinalHead<'_>,
12538) -> PrMessage {
12539    let summary = state
12540        .candidates
12541        .iter()
12542        .find(|c| c.label == winner)
12543        .map(|c| c.summary.as_str())
12544        .unwrap_or_default();
12545    // The fallback is the operator's own words and may not be English; GitHub
12546    // text always is, so a non-English task gets a neutral title instead.
12547    let review = is_review_run(state);
12548    let title = if review {
12549        facts
12550            .and_then(|f| {
12551                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12552            })
12553            .or_else(|| review_title(state))
12554            .or_else(|| {
12555                state
12556                    .candidates
12557                    .iter()
12558                    .find(|c| c.label == winner)
12559                    .filter(|c| !c.branch.starts_with("magi/"))
12560                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12561            })
12562            .unwrap_or_else(|| neutral_title(state, winner))
12563    } else {
12564        summary_title(summary).unwrap_or_else(|| {
12565            english_title(
12566                &state.instruction,
12567                PR_TITLE_MAX,
12568                state.config.graph.github_english(),
12569            )
12570            .unwrap_or_else(|| neutral_title(state, winner))
12571        })
12572    };
12573
12574    let mut body = String::new();
12575    let what = summary_without_title(summary);
12576    if !what.is_empty() {
12577        body.push_str("## Summary\n\n");
12578        body.push_str(&what);
12579        body.push_str("\n\n");
12580    }
12581
12582    // Review history uses structured facts across all rounds, never reports.
12583    body.push_str(&crate::pr_outcome::review_rounds(state));
12584    body.push_str(&crate::pr_outcome::final_verification(state, head));
12585    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12586
12587    let open = state.open_findings();
12588    if !open.is_empty() {
12589        body.push_str("## Open review findings\n\n");
12590        for f in &open {
12591            body.push_str(&format!(
12592                "- `{}` [{:?}] {}\n",
12593                f.id, f.severity, "Context retained in local review records."
12594            ));
12595        }
12596        body.push('\n');
12597    }
12598
12599    if let Some(fix) = fix
12600        && !fix.rejected.is_empty()
12601    {
12602        body.push_str("## Declined by the fixer\n\n");
12603        for r in &fix.rejected {
12604            body.push_str(&format!(
12605                "- `{}`: {}\n",
12606                r.id, "Reason retained in local review records."
12607            ));
12608        }
12609        body.push('\n');
12610    }
12611
12612    if review {
12613        // The review prompt is not the task; list what the branch carries.
12614        body.push_str("## Commits under review\n\n");
12615        if let Some(facts) = facts {
12616            let mut left = COMMITS_MAX_CHARS;
12617            for (i, (subject, text)) in facts.commits.iter().enumerate() {
12618                let mut entry = format!("- {}\n", markdown_safe(subject));
12619                for l in markdown_safe(text).lines() {
12620                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
12621                }
12622                if left == 0 {
12623                    body.push_str(&format!(
12624                        "- ... {} more commit(s)\n",
12625                        facts.commits.len() - i
12626                    ));
12627                    break;
12628                }
12629                if entry.len() > left {
12630                    // Even the first commit is cut: one huge body must not
12631                    // push the whole description past GitHub's limit.
12632                    let mut end = left;
12633                    while !entry.is_char_boundary(end) {
12634                        end -= 1;
12635                    }
12636                    entry.truncate(end);
12637                    entry.push_str("\n  ... (truncated)\n");
12638                    left = 0;
12639                } else {
12640                    left -= entry.len();
12641                }
12642                body.push_str(&entry);
12643            }
12644            if !facts.stat.trim().is_empty() {
12645                body.push_str(&format!(
12646                    "\n## Diff stat\n\n```\n{}\n```\n",
12647                    markdown_safe(facts.stat.trim())
12648                ));
12649            }
12650        } else {
12651            match &state.reviewed_commits {
12652                Some(subjects) => {
12653                    for s in subjects {
12654                        body.push_str(&format!("- {}\n", s.trim()));
12655                    }
12656                }
12657                None => {
12658                    // An older run kept only the prompt, with the commit list
12659                    // after its first paragraph.
12660                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
12661                    body.push_str(rest.trim());
12662                    body.push('\n');
12663                }
12664            }
12665        }
12666    } else {
12667        let task = state.instruction.trim();
12668        let task = if task.is_empty() {
12669            "(empty task)"
12670        } else {
12671            task
12672        };
12673        body.push_str(&format!(
12674            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
12675            task.replace("</details>", "&lt;/details&gt;")
12676        ));
12677    }
12678
12679    body.push_str(&format!(
12680        "\n---\nmagi:run/{} magi:candidate-{}\n",
12681        state.id,
12682        winner.to_ascii_lowercase()
12683    ));
12684
12685    PrMessage { title, body }
12686}
12687
12688/// Source imports have already been prepared from structured inputs. Legacy
12689/// inferred references must never instruct a seat to reconstruct those imports.
12690fn seeded_instruction(state: &RunState) -> String {
12691    let mut inputs = Vec::new();
12692    if let Some(start) = &state.sources.start_from {
12693        inputs.push(format!(
12694            "Start from `{}` at `{}` (including ancestry).",
12695            start.token, start.sha
12696        ));
12697    }
12698    for pick in &state.sources.pick_commits {
12699        inputs.push(format!(
12700            "Pick `{}` at `{}` (single commit).",
12701            pick.token, pick.sha
12702        ));
12703    }
12704    if inputs.is_empty() {
12705        state.instruction.clone()
12706    } else {
12707        format!(
12708            "{}\n\n# Explicit source inputs already prepared\n\n{}",
12709            state.instruction,
12710            inputs.join("\n")
12711        )
12712    }
12713}
12714
12715/// Does the winner have no commits ahead of the base it would land on?
12716/// Any failure to find out reads as "not empty": the merge then behaves as it
12717/// always did rather than refusing on a guess.
12718async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
12719    let base = &state.base_branch;
12720    let mut against = base.clone();
12721    if mode == MergeMode::Pr {
12722        // A pull request lands on the remote's base, never the local branch
12723        // of the same name: if that cannot be read, "not empty" is the safe
12724        // answer.
12725        let remote = &state.config.merge.remote;
12726        let tracking = format!("{remote}/{base}");
12727        let fetched = git::fetch(repo, remote, base).await;
12728        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
12729            against = tracking;
12730        } else {
12731            return false;
12732        }
12733    }
12734    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
12735}
12736
12737/// Why nothing was opened for an empty winner, with what the task's own
12738/// references resolved to.
12739fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
12740    let mut detail = format!(
12741        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
12742         nothing to open a pull request for"
12743    );
12744    detail.push_str(
12745        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
12746    );
12747    detail
12748}
12749
12750/// What the `Pr` merge does once it knows whether the branch already has an
12751/// open pull request.
12752#[derive(Debug, PartialEq, Eq)]
12753enum PrPlan {
12754    Create,
12755    Adopt { url: String, title: String },
12756    Stop(String),
12757}
12758
12759/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
12760/// a failed lookup -> stop with the real reason. Never guesses.
12761fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
12762    match found {
12763        Ok(land::OpenPr::None) => PrPlan::Create,
12764        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
12765        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
12766            "several open pull requests exist for this branch, not picking one: {}",
12767            urls.join(" ")
12768        )),
12769        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
12770    }
12771}
12772
12773/// `gh pr create`, returning the PR url.
12774pub(crate) async fn gh_pr_create(
12775    cwd: &Path,
12776    base: &str,
12777    head: &str,
12778    title: &str,
12779    body: &str,
12780) -> Result<String> {
12781    let out = tokio::process::Command::new("gh")
12782        .args([
12783            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
12784        ])
12785        .current_dir(cwd)
12786        .quiet()
12787        .stdin(std::process::Stdio::null())
12788        .output()
12789        .await
12790        .context("spawn gh")?;
12791    if out.status.success() {
12792        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
12793    } else {
12794        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
12795    }
12796}
12797
12798/// Never force deletion of uncommitted work, including unknown legacy trees.
12799async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
12800    match std::fs::symlink_metadata(path) {
12801        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
12802            // A previous cleanup may have removed the tree before a branch
12803            // deletion or state save failed. Unregister it if still present.
12804            git::worktree_remove_clean(repo, path).await?;
12805            return Ok(());
12806        }
12807        Err(e) => return Err(e.into()),
12808        Ok(_) => {}
12809    }
12810    if !git::is_clean(path).await? {
12811        bail!(
12812            "worktree has uncommitted work; refusing removal: {}",
12813            path.display()
12814        );
12815    }
12816    if !git::worktree_remove_clean(repo, path).await? {
12817        bail!("worktree removal refused: {}", path.display());
12818    }
12819    Ok(())
12820}
12821
12822/// Tear a run's worktrees and branches down.
12823///
12824/// `home` is where the updated `run.json` is saved (via
12825/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
12826/// a housekeeping pass already has its own honest `home` handed to it, and
12827/// falling through to the global here would write back through whichever
12828/// directory some other process or test pinned into that `OnceLock` first,
12829/// not the one the caller actually resolved its `runs` and `state` from.
12830pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
12831    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
12832        bail!("refusing fold: {reason}");
12833    }
12834    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
12835        bail!("candidate preservation incomplete; refusing cleanup");
12836    }
12837    // Serialize ordinary folding with the same starts/resumes as target-only
12838    // eviction. Otherwise a second janitor could remove the whole worktree
12839    // around a live output borrower even though target eviction was fenced.
12840    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
12841    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
12842    let _driver = crate::driver::maintenance(home, &state.id)?;
12843    let mut output_guards = Vec::new();
12844    for entry in std::fs::read_dir(state.worktree_root())
12845        .into_iter()
12846        .flatten()
12847        .flatten()
12848    {
12849        if entry.path().join(".git").is_file() {
12850            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
12851        }
12852    }
12853    let repo = state.repo.clone();
12854    // CLI, Web and automatic cleanup all enter here, independently of
12855    // Runner::fold_losers. Protect deletion and shared config release too.
12856    let wait = repository_wait(state, None);
12857    let _coordination = match coordinate_operation(state, &repo, &wait).await {
12858        Ok(lease) => lease,
12859        Err(e) => {
12860            state.save_under(home)?;
12861            return Err(e);
12862        }
12863    };
12864    // Confirmed merges dispose of their owned sources after durable evidence
12865    // capture. Preflight every source before removing anything.
12866    let _source_outputs = if state.status == RunStatus::Merged
12867        && state.merge.as_ref().is_some_and(|merge| merge.ok)
12868    {
12869        verify_merged_fold_sources(state, drop_winner, home).await?
12870    } else {
12871        Vec::new()
12872    };
12873    let root = state.worktree_root();
12874    let winner = state.tally.as_ref().map(|t| t.winner);
12875    let mut removed = Vec::new();
12876
12877    for i in 0..state.candidates.len() {
12878        let c = state.candidates[i].clone();
12879        let is_winner = Some(c.label) == winner;
12880        if is_winner && !drop_winner {
12881            continue;
12882        }
12883        let existed = c.worktree.exists();
12884        // A vanished directory can still have a Git registration that keeps
12885        // its branch checked out. Remove that registration before the branch.
12886        remove_preserved_worktree(&repo, &c.worktree).await?;
12887        if existed {
12888            removed.push(c.worktree.to_string_lossy().into_owned());
12889        }
12890        // A branch handed to a later run (and its pull request) is not this
12891        // run's to delete.
12892        let handed_over = state.released_branches.contains(&c.branch);
12893        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12894            if !git::branch_delete(&repo, &c.branch).await? {
12895                bail!("candidate branch removal refused");
12896            }
12897            removed.push(c.branch.clone());
12898        }
12899        state.candidates[i].folded = true;
12900    }
12901
12902    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12903        let path = name.path();
12904        let keep = !drop_winner
12905            && winner.is_some_and(|w| {
12906                path.file_name()
12907                    .is_some_and(|n| n == format!("cand-{w}").as_str())
12908            });
12909        if keep {
12910            continue;
12911        }
12912        remove_preserved_worktree(&repo, &path).await?;
12913        removed.push(path.to_string_lossy().into_owned());
12914    }
12915
12916    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
12917    // judge worktrees, so once the loop above has cleared all of them out,
12918    // the parent is a bare directory nobody else was ever going to remove -
12919    // git only ever managed what was inside it. Left alone, one of these
12920    // accumulates per fully-folded run; the operator's own machine had 74.
12921    // `remove_if_empty` re-checks rather than assuming: a run whose winner
12922    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
12923    // so does anything a run never claimed that happens to share the bay.
12924    remove_if_empty(&root);
12925
12926    if state.enabled_worktree_config && drop_winner {
12927        // A release, not a raw disable: some sibling run in this repository
12928        // may still hold its own reference (see `git::acquire_worktree_config`),
12929        // and only the last release actually turns the setting back off.
12930        git::release_worktree_config(&repo).await.ok();
12931        state.enabled_worktree_config = false;
12932    }
12933    state.save_under(home)?;
12934    Ok(removed)
12935}
12936
12937async fn verify_merged_fold_sources(
12938    state: &RunState,
12939    drop_winner: bool,
12940    home: &Path,
12941) -> Result<Vec<std::fs::File>> {
12942    let winner = state.tally.as_ref().map(|t| t.winner);
12943    let mut heads = BTreeSet::new();
12944    for candidate in &state.candidates {
12945        if Some(candidate.label) == winner && !drop_winner {
12946            continue;
12947        }
12948        anyhow::ensure!(
12949            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12950            "candidate recovery source is outside the inventoried run root: {}",
12951            candidate.worktree.display()
12952        );
12953        match std::fs::symlink_metadata(&candidate.worktree) {
12954            Ok(_) => {}
12955            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12956            Err(error) => return Err(error.into()),
12957        }
12958        if !state.released_branches.contains(&candidate.branch)
12959            && git::branch_exists(&state.repo, &candidate.branch).await?
12960        {
12961            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12962        }
12963    }
12964    let mut cargo_guards = Vec::new();
12965    let entries = match std::fs::read_dir(state.worktree_root()) {
12966        Ok(entries) => Some(entries),
12967        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12968        Err(error) => return Err(error.into()),
12969    };
12970    for entry in entries.into_iter().flatten() {
12971        let path = entry?.path();
12972        let metadata = std::fs::symlink_metadata(&path)?;
12973        anyhow::ensure!(
12974            metadata.is_dir() && !metadata.file_type().is_symlink(),
12975            "non-directory or symlink recovery source retained: {}",
12976            path.display()
12977        );
12978        if !drop_winner
12979            && winner.is_some_and(|w| {
12980                path.file_name()
12981                    .is_some_and(|name| name == format!("cand-{w}").as_str())
12982            })
12983        {
12984            continue;
12985        }
12986        anyhow::ensure!(
12987            path.join(".git").is_file(),
12988            "unregistered recovery source retained: {}",
12989            path.display()
12990        );
12991        anyhow::ensure!(
12992            git::git(
12993                &path,
12994                &[
12995                    "status",
12996                    "--porcelain",
12997                    "--untracked-files=all",
12998                    "--ignored",
12999                    "--",
13000                    ".",
13001                    ":(exclude)target"
13002                ]
13003            )
13004            .await?
13005            .is_empty(),
13006            "source changes or ignored recovery files retained: {}",
13007            path.display()
13008        );
13009        match std::fs::symlink_metadata(path.join("target")) {
13010            Ok(_) => {
13011                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13012            }
13013            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13014            Err(error) => return Err(error.into()),
13015        }
13016        for checkpoint in [
13017            "rebase-merge",
13018            "rebase-apply",
13019            "sequencer",
13020            "CHERRY_PICK_HEAD",
13021            "MERGE_HEAD",
13022            "REBASE_HEAD",
13023            "REVERT_HEAD",
13024            "BISECT_LOG",
13025            "AUTO_MERGE",
13026            "MERGE_MSG",
13027        ] {
13028            let location = git::git(
13029                &path,
13030                &[
13031                    "rev-parse",
13032                    "--path-format=absolute",
13033                    "--git-path",
13034                    checkpoint,
13035                ],
13036            )
13037            .await?;
13038            anyhow::ensure!(
13039                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13040                if e.kind() == std::io::ErrorKind::NotFound),
13041                "Git recovery checkpoint retained: {checkpoint}"
13042            );
13043        }
13044        heads.insert(git::rev_parse(&path, "HEAD").await?);
13045    }
13046    crate::fold_evidence::preserve(state, home, &heads).await?;
13047
13048    Ok(cargo_guards)
13049}
13050
13051/// Remove `dir` if it exists and has nothing in it.
13052///
13053/// Best-effort and silent by design: a directory that is not empty (a run
13054/// whose winner is still parked there, a stray file some other process left)
13055/// is exactly the case this must refuse, and a directory that is already gone
13056/// is not a failure worth reporting either. `std::fs::remove_dir` itself
13057/// already refuses a non-empty directory, so the emptiness check below is
13058/// belt, not suspenders - it is what keeps this from ever attempting the
13059/// removal in the case that matters, rather than trusting `remove_dir`'s
13060/// error path to have no side effects if it ever changed.
13061fn remove_if_empty(dir: &Path) {
13062    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13063        std::fs::remove_dir(dir).ok();
13064    }
13065}
13066
13067/// Severity of the worst open finding in the last review round, for reporting.
13068pub fn worst_open(state: &RunState) -> Option<Severity> {
13069    state
13070        .reviews
13071        .last()?
13072        .reviews
13073        .iter()
13074        .flat_map(|r| r.findings.iter())
13075        .map(|f| f.severity)
13076        .max()
13077}
13078
13079/// `[worktree] setup` hides its products per worktree, which needs
13080/// `extensions.worktreeConfig`. Taken once per run and released with the run's
13081/// worktrees, like the hook's; every place that runs setup calls this first.
13082async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13083    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13084        git::acquire_worktree_config(repo).await?;
13085        state.enabled_worktree_config = true;
13086    }
13087    Ok(())
13088}
13089
13090#[cfg(test)]
13091mod tests {
13092    #[tokio::test]
13093    async fn launcher_base_fetch_requires_remote_success() {
13094        let tmp = tempfile::tempdir().unwrap();
13095        let repo = tmp.path().join("repo");
13096        let remote = tmp.path().join("remote");
13097        std::fs::create_dir(&repo).unwrap();
13098        std::fs::create_dir(&remote).unwrap();
13099        git::git(&remote, &["init", "--bare", "-b", "main"])
13100            .await
13101            .unwrap();
13102        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13103        git::git(
13104            &repo,
13105            &[
13106                "-c",
13107                "user.name=Fixture",
13108                "-c",
13109                "user.email=fixture@example.invalid",
13110                "commit",
13111                "--allow-empty",
13112                "-m",
13113                "fixture",
13114            ],
13115        )
13116        .await
13117        .unwrap();
13118        git::git(
13119            &repo,
13120            &["remote", "add", "origin", remote.to_str().unwrap()],
13121        )
13122        .await
13123        .unwrap();
13124        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13125        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13126        assert_eq!(
13127            resolve_base(&repo, "main", "origin", None)
13128                .await
13129                .unwrap()
13130                .commit,
13131            head
13132        );
13133        #[cfg(unix)]
13134        {
13135            let policy: crate::config::Coordination = toml::from_str(
13136                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13137            ).unwrap();
13138            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13139            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13140            let held = crate::repo_lock::SeatLease::operation_with(
13141                &repo,
13142                Some(&policy),
13143                &crate::repo_lock::WaitPolicy::default(),
13144            )
13145            .await
13146            .unwrap();
13147            assert!(held.is_some());
13148            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13149            tokio::pin!(fetch);
13150            assert!(
13151                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13152                    .await
13153                    .is_err()
13154            );
13155            assert!(
13156                !repo.join(".git/FETCH_HEAD").exists(),
13157                "no fetch before admission"
13158            );
13159            drop(held);
13160            let fetched = fetch.await.unwrap();
13161            assert_eq!(fetched.commit, head);
13162            assert_eq!(fetched.remote, "origin");
13163            assert_eq!(fetched.branch, "main");
13164        }
13165        // A surviving tracking ref is not successful fetch evidence.
13166        git::git(
13167            &repo,
13168            &[
13169                "remote",
13170                "set-url",
13171                "origin",
13172                tmp.path().join("missing").to_str().unwrap(),
13173            ],
13174        )
13175        .await
13176        .unwrap();
13177        let error = resolve_base(&repo, "main", "origin", None)
13178            .await
13179            .unwrap_err()
13180            .to_string();
13181        assert!(error.contains("cannot read origin/main"));
13182        assert!(error.contains("refusing to branch off"));
13183    }
13184
13185    #[test]
13186    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13187        let leaked = vec!["chore(deps): update a crate".to_owned()];
13188        let own = "fix(daemon): apply a chosen action";
13189        assert!(should_retitle("", own, &leaked));
13190        assert!(should_retitle(
13191            &format!("{REVIEW_PROMPT_OPENING} `x`"),
13192            own,
13193            &leaked
13194        ));
13195        assert!(should_retitle(
13196            "chore: land candidate A of run 1",
13197            own,
13198            &leaked
13199        ));
13200        assert!(should_retitle(
13201            "magi: candidate A (uncommitted work)",
13202            own,
13203            &leaked
13204        ));
13205        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13206        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13207        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13208    }
13209
13210    #[test]
13211    fn pr_merge_plan_creates_adopts_or_stops() {
13212        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13213        assert_eq!(
13214            pr_merge_plan(Ok(land::OpenPr::One {
13215                url: "u".into(),
13216                title: "t".into()
13217            })),
13218            PrPlan::Adopt {
13219                url: "u".into(),
13220                title: "t".into()
13221            }
13222        );
13223        let PrPlan::Stop(many) =
13224            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13225        else {
13226            panic!("many must stop");
13227        };
13228        assert!(many.contains('a') && many.contains('b'));
13229        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13230            panic!("a failed lookup must stop");
13231        };
13232        assert!(err.contains("bad token"));
13233    }
13234
13235    use super::*;
13236    use crate::run::GateStatus;
13237    use std::collections::BTreeMap;
13238    use std::time::Duration;
13239
13240    fn conductor() -> AgentSpec {
13241        AgentSpec {
13242            id: "conductor".to_owned(),
13243            kind: crate::config::AgentKind::Command,
13244            model: None,
13245            command: vec!["true".to_owned()],
13246            extra_args: Vec::new(),
13247            env: BTreeMap::new(),
13248            prompt_delivery: None,
13249            max_turn_secs: None,
13250        }
13251    }
13252
13253    fn spec(id: &str) -> AgentSpec {
13254        AgentSpec {
13255            id: id.to_owned(),
13256            kind: crate::config::AgentKind::Command,
13257            model: None,
13258            command: vec!["true".to_owned()],
13259            extra_args: Vec::new(),
13260            env: BTreeMap::new(),
13261            prompt_delivery: None,
13262            max_turn_secs: None,
13263        }
13264    }
13265
13266    fn ids(xs: &[&str]) -> BTreeSet<String> {
13267        xs.iter().map(|s| (*s).to_owned()).collect()
13268    }
13269
13270    #[test]
13271    fn pick_successor_skips_an_agent_another_seat_holds() {
13272        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
13273        let roster = [spec("a"), spec("b"), spec("c")];
13274        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13275        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13276    }
13277
13278    #[test]
13279    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13280        // The other seat started on c but was handed to d; c is free again.
13281        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13282        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13283        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13284        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13285        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13286    }
13287
13288    #[test]
13289    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13290        let roster = [spec("a"), spec("b")];
13291        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13292        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13293        let carried = ids(&["b"]);
13294        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13295        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13296    }
13297
13298    #[test]
13299    fn pick_successor_returns_none_without_any_untried_successor() {
13300        let roster = [spec("a"), spec("b")];
13301        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13302        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13303        let carried = ids(&["a"]);
13304        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13305    }
13306
13307    #[test]
13308    fn pick_successor_rescue_avoids_another_seats_occupant() {
13309        // b is a carried failure and free; a is held by the other seat.
13310        let roster = [spec("a"), spec("b"), spec("c")];
13311        let carried = ids(&["b", "c"]);
13312        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13313        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13314    }
13315
13316    #[test]
13317    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13318        let roster = [spec("a"), spec("b"), spec("c")];
13319        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13320        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13321    }
13322
13323    #[test]
13324    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13325        let roster = [spec("a"), spec("b"), spec("c")];
13326        let failed = ids(&["a", "b", "c"]);
13327        // Rescue looks at the whole roster, once per id, then runs out.
13328        let mut tried = ids(&["b"]);
13329        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13330        assert_eq!(first.id, "a");
13331        tried.insert(first.id.clone());
13332        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13333        assert_eq!(second.id, "c");
13334        tried.insert(second.id.clone());
13335        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13336    }
13337
13338    #[test]
13339    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13340        let roster = [spec("a"), spec("b")];
13341        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13342        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13343    }
13344
13345    #[test]
13346    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13347        let roster = [spec("a"), spec("b"), spec("c")];
13348        let h = SeatHistory {
13349            failed: ids(&["a"]),
13350            last_ok: Some("b".to_owned()),
13351            last_fail: None,
13352        };
13353        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13354        // A last answerer that left the roster, or later failed, is ignored.
13355        let gone = SeatHistory {
13356            last_ok: Some("zzz".to_owned()),
13357            ..h.clone()
13358        };
13359        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13360        let failed = SeatHistory {
13361            failed: ids(&["a", "b"]),
13362            last_ok: Some("b".to_owned()),
13363            last_fail: None,
13364        };
13365        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13366    }
13367
13368    #[test]
13369    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13370        let first = SeatState::new("review-1", "alpha", 7);
13371        let next = handover_seat("review-1", "gamma", 7);
13372        assert_ne!(first.claude_session, next.claude_session);
13373    }
13374
13375    #[test]
13376    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13377        let mut state = state_with_summary("x", "y");
13378        let a = handover_seat("review-1", "beta", state.next_seat_seed());
13379        let b = handover_seat("review-1", "beta", state.next_seat_seed());
13380        assert_ne!(a.claude_session, b.claude_session);
13381    }
13382
13383    #[test]
13384    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13385        let roster = [spec("a"), spec("b")];
13386        let h = SeatHistory {
13387            failed: ids(&["a", "b"]),
13388            ..SeatHistory::default()
13389        };
13390        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13391        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13392        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13393    }
13394
13395    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
13396    // fallback loop depends on to terminate: it must walk forward from the
13397    // seat's own position, never restart at the front of the roster, and it
13398    // must never hand back an id already tried, however many times that id
13399    // happens to appear.
13400
13401    #[test]
13402    fn failure_signature_ignores_numbers_and_paths() {
13403        assert_eq!(
13404            failure_signature("exited with Some(2) and no usable output"),
13405            failure_signature("exited with Some(137) and no usable output")
13406        );
13407        assert_eq!(
13408            failure_signature("process exited with no usable output (exit code 2)"),
13409            failure_signature("process exited with no usable output (exit code 137)")
13410        );
13411        for (old, new) in [
13412            (
13413                "exited with Some(2) and no usable output",
13414                "process exited with no usable output (exit code 2)",
13415            ),
13416            (
13417                "exited with None and no usable output",
13418                "process exited with no usable output (exit code unavailable)",
13419            ),
13420        ] {
13421            let previous = FailClass::Other(failure_signature(old));
13422            let current = FailClass::Other(failure_signature(new));
13423            assert_eq!(previous, current, "resumption retains the failure class");
13424            assert!(!should_hand_over(Some(&previous), &current));
13425        }
13426        assert_eq!(
13427            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13428            failure_signature("cannot open /var/x.txt: denied")
13429        );
13430        assert_ne!(failure_signature("boom"), failure_signature("bang"));
13431    }
13432
13433    #[test]
13434    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13435        let other = FailClass::Other("x".into());
13436        assert!(should_hand_over(None, &FailClass::Quota));
13437        assert!(should_hand_over(Some(&other), &FailClass::Quota));
13438        assert!(should_hand_over(
13439            Some(&FailClass::Timeout),
13440            &FailClass::Timeout
13441        ));
13442        assert!(should_hand_over(None, &other));
13443        assert!(!should_hand_over(Some(&other), &other));
13444        assert!(should_hand_over(
13445            Some(&other),
13446            &FailClass::Other("y".into())
13447        ));
13448        // A quota or timeout in between ends the run of identical failures.
13449        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13450        assert!(should_hand_over(Some(&FailClass::Quota), &other));
13451    }
13452
13453    #[test]
13454    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13455        let a = SeatState::new("judge-1", "alpha", 7);
13456        let b = handover_seat("judge-1", "beta", 7);
13457        assert_ne!(a.claude_session, b.claude_session);
13458        assert_eq!(b.turns, 0);
13459    }
13460
13461    #[test]
13462    fn a_timeout_is_classified_apart_from_other_failures() {
13463        assert_eq!(
13464            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13465            Some(FailClass::Timeout)
13466        );
13467        assert!(matches!(
13468            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13469            Some(FailClass::Other(_))
13470        ));
13471    }
13472
13473    #[test]
13474    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13475        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13476        let tried = BTreeSet::from(["beta".to_owned()]);
13477        // beta sits at index 1; the next candidate is gamma, never alpha —
13478        // which is very likely a different candidate slot's own agent.
13479        let next = next_untried_in_roster(&roster, 1, &tried);
13480        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13481    }
13482
13483    #[test]
13484    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13485        let roster = vec![spec("alpha"), spec("beta")];
13486        let tried = BTreeSet::from(["beta".to_owned()]);
13487        // beta is the roster's last entry: nothing follows it, and alpha —
13488        // earlier in the roster, almost certainly a different candidate
13489        // slot's own agent — must not be reached by wrapping back to it.
13490        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13491    }
13492
13493    #[test]
13494    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13495        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13496        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13497        // beta (index 1) and gamma (index 2, the only entry after it) have
13498        // both been tried; alpha (index 0) never has, but it comes before
13499        // beta's own position, so there is nothing further for this seat.
13500        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13501    }
13502
13503    #[test]
13504    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13505        let roster = vec![spec("a"), spec("a"), spec("b")];
13506        let tried = BTreeSet::from(["a".to_owned()]);
13507        let next = next_untried_in_roster(&roster, 0, &tried);
13508        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13509    }
13510
13511    #[test]
13512    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13513        let roster = vec![spec("a"), spec("b")];
13514        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13515        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13516    }
13517
13518    #[test]
13519    fn remove_if_empty_only_ever_takes_a_bare_directory() {
13520        let dir = tempfile::tempdir().unwrap();
13521        let bay = dir.path().join("ffff");
13522
13523        // Not there yet: nothing to do, nothing to panic on.
13524        remove_if_empty(&bay);
13525        assert!(!bay.exists());
13526
13527        // Something still inside - the winner's worktree, or a stray file -
13528        // keeps the directory standing.
13529        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13530        remove_if_empty(&bay);
13531        assert!(bay.exists(), "non-empty directory must survive");
13532
13533        // Once the last entry is gone, so is the directory itself.
13534        std::fs::remove_dir(bay.join("cand-A")).unwrap();
13535        remove_if_empty(&bay);
13536        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13537    }
13538
13539    // `round_is_clean` is the exact decision this task fixed: a round with a
13540    // seat that never answered must not read the same as a round every seat
13541    // actually reviewed. These are deterministic and process-free by design —
13542    // the equivalent end-to-end check (a real reviewer timing out under a
13543    // live graph run) is a genuine race against wall-clock contention, and a
13544    // spawn slow enough to blow even a generous budget under a loaded test
13545    // run must not turn this specific regression check flaky.
13546
13547    #[test]
13548    fn a_full_panel_that_found_nothing_is_clean() {
13549        assert!(round_is_clean(
13550            0,
13551            true,
13552            2,
13553            2,
13554            0,
13555            IncompleteReviewPolicy::Block
13556        ));
13557    }
13558
13559    #[test]
13560    fn a_missing_seat_is_never_clean_under_the_default_policy() {
13561        assert!(!round_is_clean(
13562            0,
13563            true,
13564            1,
13565            2,
13566            0,
13567            IncompleteReviewPolicy::Block
13568        ));
13569    }
13570
13571    #[test]
13572    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13573        assert!(!round_is_clean(
13574            1,
13575            true,
13576            1,
13577            2,
13578            0,
13579            IncompleteReviewPolicy::Warn
13580        ));
13581    }
13582
13583    #[test]
13584    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13585        assert!(round_is_clean(
13586            0,
13587            true,
13588            1,
13589            2,
13590            0,
13591            IncompleteReviewPolicy::Warn
13592        ));
13593    }
13594
13595    #[test]
13596    fn a_full_panel_with_an_open_finding_is_not_clean() {
13597        assert!(!round_is_clean(
13598            1,
13599            true,
13600            2,
13601            2,
13602            0,
13603            IncompleteReviewPolicy::Block
13604        ));
13605    }
13606
13607    #[test]
13608    fn a_full_panel_with_a_red_e2e_is_not_clean() {
13609        assert!(!round_is_clean(
13610            0,
13611            false,
13612            2,
13613            2,
13614            0,
13615            IncompleteReviewPolicy::Block
13616        ));
13617    }
13618
13619    // The stall this task closes: under the default `block` policy, a seat
13620    // missing only because it was rate limited must not force a wait for a
13621    // session limit that will not lift by the next round. `round_is_clean`
13622    // is where that quorum carve-out lives; the review loop around it never
13623    // changes what a reviewer's vote or a finding's severity means.
13624
13625    #[test]
13626    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13627        // 1 of 2 answered, and the one missing was quota'd — the exact
13628        // "review-2 rate limited (quota)" shape from the field report.
13629        assert!(round_is_clean(
13630            0,
13631            true,
13632            1,
13633            2,
13634            1,
13635            IncompleteReviewPolicy::Block
13636        ));
13637    }
13638
13639    #[test]
13640    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13641        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
13642        // not a quota loss (`quota_missing` stays 0) — worth another try.
13643        assert!(!round_is_clean(
13644            0,
13645            true,
13646            1,
13647            2,
13648            0,
13649            IncompleteReviewPolicy::Block
13650        ));
13651    }
13652
13653    #[test]
13654    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
13655        assert!(!round_is_clean(
13656            1,
13657            true,
13658            1,
13659            2,
13660            1,
13661            IncompleteReviewPolicy::Block
13662        ));
13663        assert!(!round_is_clean(
13664            0,
13665            false,
13666            1,
13667            2,
13668            1,
13669            IncompleteReviewPolicy::Block
13670        ));
13671    }
13672
13673    #[test]
13674    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
13675        // Every seat quota'd, nobody answered: there is no panel to decide
13676        // on, so this must fall through to the existing block-and-retry
13677        // fallback rather than call an unreviewed patch clean.
13678        assert!(!round_is_clean(
13679            0,
13680            true,
13681            0,
13682            2,
13683            2,
13684            IncompleteReviewPolicy::Block
13685        ));
13686    }
13687
13688    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
13689        CommandOutcome {
13690            command: "test".to_owned(),
13691            code,
13692            output_tail: String::new(),
13693            duration_ms: 0,
13694            resource_blocked,
13695        }
13696    }
13697
13698    #[test]
13699    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
13700        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
13701        assert!(
13702            !verify_inconclusive(&[outcome(Some(1), false)]),
13703            "an ordinary failure is still evidence about the patch"
13704        );
13705        assert!(verify_inconclusive(&[outcome(None, true)]));
13706        assert!(
13707            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
13708            "one inconclusive outcome taints the whole batch"
13709        );
13710        assert!(!verify_inconclusive(&[]));
13711    }
13712
13713    #[tokio::test]
13714    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
13715        // Alive for the first two checks, then dead - confirms the loop
13716        // actually re-polls rather than deciding once and sleeping out the
13717        // ceiling regardless.
13718        let calls = std::sync::atomic::AtomicUsize::new(0);
13719        let started = Instant::now();
13720        wait_for_pids_with(
13721            &[123],
13722            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
13723            Duration::from_millis(5),
13724            Duration::from_secs(5),
13725        )
13726        .await;
13727        assert!(
13728            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
13729            "must keep checking rather than deciding on the first answer"
13730        );
13731        assert!(
13732            started.elapsed() < Duration::from_secs(1),
13733            "must return the moment it is confirmed dead, not wait out the ceiling"
13734        );
13735    }
13736
13737    #[tokio::test]
13738    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
13739        let started = Instant::now();
13740        wait_for_pids_with(
13741            &[123],
13742            |_| true, // never reports dead
13743            Duration::from_millis(5),
13744            Duration::from_millis(30),
13745        )
13746        .await;
13747        let elapsed = started.elapsed();
13748        assert!(
13749            elapsed >= Duration::from_millis(30),
13750            "must not give up before its own ceiling: {elapsed:?}"
13751        );
13752        assert!(
13753            elapsed < Duration::from_secs(1),
13754            "must not wait past its own ceiling either: {elapsed:?}"
13755        );
13756    }
13757
13758    #[tokio::test]
13759    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
13760        let started = Instant::now();
13761        wait_for_pids_with(
13762            &[],
13763            |_| true,
13764            Duration::from_secs(5),
13765            Duration::from_secs(5),
13766        )
13767        .await;
13768        assert!(
13769            started.elapsed() < Duration::from_millis(200),
13770            "an empty pid list has nothing to confirm"
13771        );
13772    }
13773
13774    // `review_conclusion` is the exact decision the review hand-off task
13775    // fixed: a round budget spent (or a tree that stopped moving) must not
13776    // collapse into `Blocked` regardless of what verification actually
13777    // said. Deterministic and process-free for the same reason the
13778    // `round_is_clean` family above is.
13779    fn review_round(
13780        clean: bool,
13781        blocking: usize,
13782        answered: usize,
13783        expected: usize,
13784        progressed: bool,
13785        e2e_ok: bool,
13786    ) -> ReviewRound {
13787        ReviewRound {
13788            round: 1,
13789            head: "h".to_owned(),
13790            verified_head: None,
13791            verified_at: None,
13792            reviews: Vec::new(),
13793            e2e: vec![CommandOutcome {
13794                command: "test".to_owned(),
13795                code: Some(if e2e_ok { 0 } else { 1 }),
13796                output_tail: String::new(),
13797                duration_ms: 0,
13798                resource_blocked: false,
13799            }],
13800            verify_retried: false,
13801            e2e_deferred: false,
13802            e2e_defer_reason: None,
13803            fix: None,
13804            blocking,
13805            answered,
13806            expected,
13807            clean,
13808            progressed,
13809            vote_split: false,
13810            reconsideration: Vec::new(),
13811            verdict: None,
13812        }
13813    }
13814
13815    #[test]
13816    fn review_conclusion_is_none_when_nothing_has_run() {
13817        assert_eq!(review_conclusion(&[], 3), None);
13818    }
13819
13820    #[test]
13821    fn review_conclusion_is_none_while_rounds_remain() {
13822        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
13823        assert_eq!(review_conclusion(&rounds, 3), None);
13824    }
13825
13826    #[test]
13827    fn review_conclusion_is_gating_once_a_round_is_clean() {
13828        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
13829        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
13830    }
13831
13832    #[test]
13833    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
13834        let rounds = vec![
13835            review_round(false, 1, 2, 2, true, true),
13836            review_round(false, 1, 2, 2, true, true),
13837        ];
13838        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
13839    }
13840
13841    #[test]
13842    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
13843        let rounds = vec![
13844            review_round(false, 1, 2, 2, true, true),
13845            review_round(false, 1, 2, 2, true, false),
13846        ];
13847        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
13848    }
13849
13850    #[test]
13851    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
13852        // Magi never got a command to run against this round's own head — a
13853        // resource-blocked attempt, not a red one — so this must never
13854        // settle on `Blocked` the way a genuine e2e failure would. `None`
13855        // here is what tells `Runner::review_loop` to retry the check
13856        // itself rather than trust this cheap recomputation with a verdict
13857        // it cannot actually produce.
13858        let mut blocked = review_round(false, 1, 2, 2, true, false);
13859        blocked.e2e[0].resource_blocked = true;
13860        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
13861        assert_eq!(review_conclusion(&rounds, 2), None);
13862    }
13863
13864    #[test]
13865    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
13866        // Missing input, not a verified tree — never a hand-off candidate.
13867        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
13868        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
13869    }
13870
13871    #[test]
13872    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
13873        let rounds = vec![
13874            review_round(false, 1, 2, 2, false, true),
13875            review_round(false, 1, 2, 2, false, true),
13876        ];
13877        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
13878    }
13879
13880    fn secs(n: u64) -> Duration {
13881        Duration::from_secs(n)
13882    }
13883
13884    /// A throwaway repo with one commit on `main`, for tests that need `merge`
13885    /// to make real (and, if it runs at all, real*ly fail*) git calls.
13886    #[tokio::test]
13887    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13888        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
13889        if std::env::var_os(CHILD).is_none() {
13890            // Other tests reconcile every run in the process-global home,
13891            // briefly taking owner locks. This ancestry test expects an idle
13892            // predecessor, so give it its own process and pinned test home.
13893            let output = std::process::Command::new(std::env::current_exe().unwrap())
13894                .args([
13895                    "--exact",
13896                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
13897                    "--nocapture",
13898                ])
13899                .env(CHILD, "1")
13900                .env_remove("MAGI_WORKER_HOME")
13901                .env_remove("MAGI_WORKER_ID")
13902                .output()
13903                .unwrap();
13904            assert!(
13905                output.status.success()
13906                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
13907                "{}\n{}",
13908                String::from_utf8_lossy(&output.stdout),
13909                String::from_utf8_lossy(&output.stderr)
13910            );
13911            return;
13912        }
13913        ask_test_home();
13914        let tmp = tempfile::tempdir().unwrap();
13915        let repo = tmp.path().join("repo");
13916        std::fs::create_dir_all(&repo).unwrap();
13917        init_repo(&repo);
13918        let g = |args: &[&str]| {
13919            let out = std::process::Command::new("git")
13920                .args(args)
13921                .current_dir(&repo)
13922                .quiet()
13923                .output()
13924                .unwrap();
13925            assert!(
13926                out.status.success(),
13927                "{}",
13928                String::from_utf8_lossy(&out.stderr)
13929            );
13930        };
13931        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13932        g(&["checkout", "-b", "historical"]);
13933        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13934        g(&["add", "."]);
13935        g(&["commit", "-m", "old alternative"]);
13936        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13937        g(&["checkout", "-b", "magi/task-selected", &base]);
13938        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13939        g(&["add", "."]);
13940        g(&["commit", "-m", "measurement infrastructure"]);
13941        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13942        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13943        g(&["add", "."]);
13944        g(&["commit", "-m", "dependent implementation"]);
13945        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13946        g(&["checkout", "main"]);
13947        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13948        g(&["add", "."]);
13949        g(&["commit", "-m", "unrelated base change"]);
13950        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13951        let origin = tmp.path().join("origin.git");
13952        g(&[
13953            "clone",
13954            "--bare",
13955            repo.to_str().unwrap(),
13956            origin.to_str().unwrap(),
13957        ]);
13958        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13959        g(&["fetch", "origin"]);
13960        let selection = crate::pr_recovery::Selection {
13961            url: "https://example.test/project/pull/1".into(),
13962            branch: "magi/task-selected".into(),
13963            head: head.clone(),
13964            source_head: None,
13965            source_run: None,
13966            source_proof: None,
13967            disposition: Some(crate::pr_recovery::Disposition::Review),
13968            instruction: "preserve measured behavior".into(),
13969            approved_at: jiff::Timestamp::now(),
13970        };
13971        let mut template = runner_at(RunStatus::Reviewing);
13972        template.state.repo = repo.clone();
13973        template.state.base_commit = current.clone();
13974        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13975        template.state.origin = Some(crate::run::Origin::queue("selected"));
13976        template.state.pr_head = Some(selection.branch.clone());
13977        template.state.pr_recovery = Some(selection.clone());
13978        let mut runner =
13979            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13980                .await
13981                .unwrap();
13982        let wt = runner.state.winner().unwrap().worktree.clone();
13983        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13984        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13985        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13986        assert!(runner.state.seeds.is_empty());
13987        assert_eq!(runner.state.instruction, "preserve measured behavior");
13988        runner.sync_to_base().await.unwrap();
13989        assert_eq!(
13990            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13991            "prerequisite"
13992        );
13993        assert_eq!(
13994            std::fs::read_to_string(wt.join("implementation")).unwrap(),
13995            "measured implementation"
13996        );
13997        assert_eq!(
13998            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13999            "current main change"
14000        );
14001        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14002        assert_eq!(
14003            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14004            head
14005        );
14006
14007        // A failed selected-source setup stays inspectable, never resumable as a new competition.
14008        let mut partial = runner_at(RunStatus::Reviewing);
14009        partial.state.repo = repo.clone();
14010        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14011        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14012            run: Some("exit 1".into()),
14013            ..Default::default()
14014        }];
14015        let selection = runner.state.pr_recovery.clone().unwrap();
14016        partial.state.pr_recovery = Some(selection.clone());
14017        partial.state.pr_head = Some(selection.branch.clone());
14018        partial.state.origin = Some(crate::run::Origin::queue("selected"));
14019        let id = partial.state.id.clone();
14020        let error = match Runner::open_selected(
14021            &repo,
14022            partial.state,
14023            partial.roles,
14024            1,
14025            runner.state.base_commit.clone(),
14026            selection,
14027        )
14028        .await
14029        {
14030            Ok(_) => panic!("setup must fail"),
14031            Err(e) => e,
14032        };
14033        assert_eq!(
14034            error
14035                .downcast_ref::<crate::run::StartupFailure>()
14036                .unwrap()
14037                .run,
14038            id
14039        );
14040        let failed = RunState::load(&id).unwrap();
14041        assert_eq!(failed.status, RunStatus::Failed);
14042        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14043        assert!(Runner::resume(&id).is_err());
14044        runner.state.status = RunStatus::Ready;
14045        runner.state.driver_exited = true;
14046        runner.state.save().unwrap();
14047        let mut retry = runner_at(RunStatus::Reviewing);
14048        retry.state.repo = repo.clone();
14049        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14050        let mut reselected = runner.state.pr_recovery.clone().unwrap();
14051        reselected.approved_at = jiff::Timestamp::now();
14052        retry.state.origin = Some(crate::run::Origin::queue("selected"));
14053        retry.state.pr_head = Some(reselected.branch.clone());
14054        retry.state.pr_recovery = Some(reselected.clone());
14055        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14056            .await
14057            .unwrap();
14058        let restarted = Runner::open_selected(
14059            &repo,
14060            retry.state,
14061            retry.roles,
14062            1,
14063            runner.state.base_commit.clone(),
14064            reselected,
14065        )
14066        .await
14067        .unwrap();
14068        assert_eq!(
14069            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14070                .await
14071                .unwrap(),
14072            head
14073        );
14074        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14075        assert_eq!(
14076            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14077            head
14078        );
14079    }
14080
14081    #[tokio::test]
14082    #[cfg(unix)]
14083    async fn fallback_and_interrupted_restoration_require_coordination() {
14084        use std::os::fd::AsRawFd as _;
14085        ask_test_home();
14086        let tmp = tempfile::tempdir().unwrap();
14087        let mut runner = registered_runner(tmp.path()).await;
14088        runner.state.config.graph.timeout_resource_wait = 1;
14089        let winner = runner.state.candidates[0].clone();
14090        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14091        let leftover = winner.worktree.join("leftover");
14092        std::fs::write(&leftover, "untrusted edits").unwrap();
14093        let next = tmp.path().join("next-fixer");
14094        let mut failed = spec("first");
14095        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14096        let mut successor = spec("second");
14097        successor.command = vec![
14098            "sh".into(),
14099            "-c".into(),
14100            format!("touch '{}'; echo done", next.display()),
14101        ];
14102        let attempts = vec![
14103            (failed.clone(), "first".into()),
14104            (successor.clone(), "second".into()),
14105        ];
14106        let lock_path = runner.state.repo.join(".git/updater.lock");
14107        let external = std::fs::File::create(&lock_path).unwrap();
14108        assert_eq!(
14109            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14110            0
14111        );
14112        let artifacts = tmp.path().join("artifacts");
14113        let build = |spec, seat| SeatJob {
14114            spec,
14115            seat,
14116            cwd: winner.worktree.clone(),
14117            prompt: "fixture".into(),
14118            timeout: Duration::from_secs(10),
14119            allow_write: true,
14120            sessions: false,
14121            artifacts: artifacts.clone(),
14122            stem: "fix".into(),
14123            handover: None,
14124        };
14125        let (_, _, out) = runner
14126            .ask_fixer_among(
14127                attempts.clone(),
14128                "graft",
14129                None,
14130                Some((&winner.worktree, &head)),
14131                build,
14132            )
14133            .await;
14134        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14135        assert!(leftover.exists(), "contention must prevent clean");
14136        assert!(!next.exists());
14137        assert!(runner.state.handovers.is_empty());
14138        // Early interrupted-repair rollback uses the same protected route.
14139        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14140            from_run: "earlier".into(),
14141            generation: 1,
14142            mode: crate::recovery::Mode::VerifyRepair,
14143            reasons: Vec::new(),
14144            reviewed_head: None,
14145            rounds: Vec::new(),
14146            open_findings: Vec::new(),
14147            evidence: None,
14148            criteria_inherited: true,
14149            repair: crate::recovery::Repair::Started {
14150                before: head.clone(),
14151            },
14152        });
14153        assert!(
14154            runner
14155                .recovery_repair(&winner)
14156                .await
14157                .unwrap_err()
14158                .is::<crate::repo_lock::ResourceBlocked>()
14159        );
14160        assert!(matches!(
14161            runner.state.recovery.as_ref().unwrap().repair,
14162            crate::recovery::Repair::Started { .. }
14163        ));
14164        assert!(leftover.exists());
14165        drop(external);
14166        runner.recovery_repair(&winner).await.unwrap();
14167        assert!(!leftover.exists());
14168        assert!(matches!(
14169            runner.state.recovery.as_ref().unwrap().repair,
14170            crate::recovery::Repair::Done { .. }
14171        ));
14172        // A hard lock-open failure also leaves HEAD and untracked work intact.
14173        std::fs::remove_file(&lock_path).unwrap();
14174        std::fs::create_dir(&lock_path).unwrap();
14175        std::fs::write(&leftover, "untrusted edits").unwrap();
14176        let (_, _, out) = runner
14177            .ask_fixer_among(
14178                attempts,
14179                "graft",
14180                None,
14181                Some((&winner.worktree, &head)),
14182                build,
14183            )
14184            .await;
14185        assert!(matches!(out, AgentOutcome::Failed(_)));
14186        assert!(leftover.exists());
14187        assert!(!next.exists());
14188        assert_eq!(
14189            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14190            head
14191        );
14192    }
14193
14194    #[cfg(unix)]
14195    async fn registered_runner(root: &Path) -> Runner {
14196        repository_runner(root, crate::config::test_coordination()).await
14197    }
14198
14199    async fn repository_runner(
14200        root: &Path,
14201        coordination: Option<crate::config::Coordination>,
14202    ) -> Runner {
14203        ask_test_home();
14204        let repo = root.join("repo");
14205        std::fs::create_dir_all(&repo).unwrap();
14206        init_repo(&repo);
14207        if coordination.is_some() {
14208            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14209        }
14210        let mut runner = runner_at(RunStatus::Prep);
14211        runner.state.repo = repo.clone();
14212        runner.state.config.coordination = coordination;
14213        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14214        runner.state.config.graph.worktree_root = Some(root.join("wt"));
14215        runner.state.config.graph.advise = false;
14216        runner.roles.implementers = vec![spec("implement")];
14217        runner.roles.judges = vec![spec("judge")];
14218        runner.prep().await.unwrap();
14219        runner
14220    }
14221
14222    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14223
14224    #[tokio::test]
14225    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14226        let tmp = tempfile::tempdir().unwrap();
14227        let mut runner = repository_runner(tmp.path(), None).await;
14228        let prompts = runner.state.config.prompts.clone();
14229        let run = runner.state.id.clone();
14230        let ctx = WaveCtx {
14231            carry_seats: false,
14232            pauses: None,
14233            run: &run,
14234            node: "advise",
14235            prompts: &prompts,
14236            cache: None,
14237            round: None,
14238        };
14239        let jobs: Vec<_> = ["alpha", "beta"]
14240            .into_iter()
14241            .map(|id| SeatJob {
14242                spec: scripted(id, AUTH_OUTAGE),
14243                seat: SeatState::new(id, id, 7),
14244                prompt: "propose".into(),
14245                cwd: runner.state.candidates[0].worktree.clone(),
14246                timeout: Duration::from_secs(10),
14247                allow_write: false,
14248                sessions: false,
14249                artifacts: agent::artifacts_dir(&runner.state.dir()),
14250                stem: id.into(),
14251                handover: None,
14252            })
14253            .collect();
14254        let mut losses = Vec::new();
14255        let results = ask_wave_with(
14256            jobs.clone(),
14257            runner.sem.clone(),
14258            2,
14259            &[],
14260            &ctx,
14261            &mut losses,
14262            &mut runner.state,
14263            &|_: &str| Ok(()),
14264        )
14265        .await;
14266        assert!(
14267            results
14268                .iter()
14269                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14270        );
14271        assert_eq!(runner.state.quota.len(), 2);
14272        assert!(
14273            runner
14274                .state
14275                .quota
14276                .iter()
14277                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14278        );
14279        assert!(!runner.state.advise_attempted);
14280        runner.stop_for_resource().unwrap();
14281        assert_eq!(runner.state.status, RunStatus::Blocked);
14282        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14283        assert_eq!(
14284            runner.state.resource_block.as_ref().unwrap().kind,
14285            "agent authentication"
14286        );
14287        // A machine loss followed by an ordinary failure is not evidence
14288        // that the whole node was unavailable, and must not earn a refund.
14289        runner.state.resource_block = None;
14290        runner.state.quota.clear();
14291        let mut mixed = jobs;
14292        mixed[1].spec.command = vec!["false".into()];
14293        let results = ask_wave_with(
14294            mixed,
14295            runner.sem.clone(),
14296            0,
14297            &[],
14298            &ctx,
14299            &mut losses,
14300            &mut runner.state,
14301            &|_: &str| Ok(()),
14302        )
14303        .await;
14304        assert!(results.iter().all(|(_, res, _)| res.is_err()));
14305        assert!(runner.state.resource_block.is_none());
14306        assert_eq!(losses.len(), 1);
14307    }
14308
14309    #[tokio::test]
14310    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14311        let tmp = tempfile::tempdir().unwrap();
14312        let mut runner = repository_runner(tmp.path(), None).await;
14313        let implementer = scripted("implement", AUTH_OUTAGE);
14314        runner.roles.implementers = vec![implementer.clone()];
14315        runner.state.config.agents = vec![implementer];
14316        runner.implement().await.unwrap();
14317        assert_eq!(runner.state.status, RunStatus::Blocked);
14318        assert_eq!(runner.state.quota.len(), 1);
14319        assert_eq!(
14320            runner.state.quota[0].cause,
14321            agent::QuotaCause::Authentication
14322        );
14323        let candidate = &runner.state.candidates[0];
14324        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14325        assert_eq!(runner.state.seats["impl-A"].turns, 1);
14326    }
14327
14328    #[tokio::test]
14329    #[cfg(unix)]
14330    async fn parked_implementer_persists_completed_work() {
14331        for registered in [false, true] {
14332            let tmp = tempfile::tempdir().unwrap();
14333            let mut runner = registered_runner(tmp.path()).await;
14334            if !registered {
14335                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14336            }
14337            let marker = tmp.path().join("started");
14338            let release = tmp.path().join("release");
14339            let mut implementer = spec("implement");
14340            implementer.command = vec![
14341                "sh".into(),
14342                "-c".into(),
14343                format!(
14344                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14345                    marker.display(),
14346                    release.display()
14347                ),
14348            ];
14349            runner.roles.implementers = vec![implementer.clone()];
14350            runner.state.config.agents = vec![implementer];
14351            let pause = runner.pause.clone();
14352            let (result, ()) = tokio::join!(runner.implement(), async {
14353                tokio::time::timeout(Duration::from_secs(10), async {
14354                    while !marker.exists() {
14355                        tokio::time::sleep(Duration::from_millis(10)).await;
14356                    }
14357                })
14358                .await
14359                .unwrap();
14360                pause.park();
14361                std::fs::write(&release, "go").unwrap();
14362            });
14363            result.unwrap();
14364            let candidate = &runner.state.candidates[0];
14365            assert!(candidate.commits > 0, "{candidate:?}");
14366            assert!(candidate.summary.contains("saved-result"));
14367            assert_eq!(candidate.files, 1);
14368            let before = candidate.commits;
14369            runner.implement().await.unwrap();
14370            assert_eq!(runner.state.candidates[0].commits, before);
14371        }
14372    }
14373
14374    #[tokio::test]
14375    #[cfg(unix)]
14376    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14377        let tmp = tempfile::tempdir().unwrap();
14378        let mut runner = registered_runner(tmp.path()).await;
14379        let winner = runner.state.candidates[0].clone();
14380        let cache = tmp.path().join("cache");
14381        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14382        runner.state.config.graph.timeout_resource_wait = 0;
14383        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14384        let crate::cache::AcquireOutcome::Acquired(held) =
14385            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14386        else {
14387            panic!("cache must be free")
14388        };
14389        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14390            from_run: "earlier".into(),
14391            generation: 1,
14392            mode: crate::recovery::Mode::VerifyRepair,
14393            reasons: Vec::new(),
14394            reviewed_head: None,
14395            rounds: Vec::new(),
14396            open_findings: Vec::new(),
14397            evidence: None,
14398            criteria_inherited: true,
14399            repair: crate::recovery::Repair::NotStarted,
14400        });
14401        runner.recovery_repair(&winner).await.unwrap();
14402        assert!(matches!(
14403            runner.state.recovery.as_ref().unwrap().repair,
14404            crate::recovery::Repair::WaitingForRestore { .. }
14405        ));
14406        drop(held);
14407        runner.state.resource_block = None;
14408        runner.state.config.graph.timeout_resource_wait = 1;
14409        runner.recovery_repair(&winner).await.unwrap();
14410        assert!(matches!(
14411            runner.state.recovery.as_ref().unwrap().repair,
14412            crate::recovery::Repair::Done { .. }
14413        ));
14414        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14415    }
14416
14417    #[tokio::test]
14418    #[cfg(unix)]
14419    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14420        use std::os::fd::AsRawFd as _;
14421        for graft in [false, true] {
14422            let tmp = tempfile::tempdir().unwrap();
14423            let mut runner = registered_runner(tmp.path()).await;
14424            runner.state.config.graph.timeout_resource_wait = 1;
14425            let winner = runner.state.candidates[0].clone();
14426            let marker = tmp.path().join("fixer-started");
14427            let mut fixer = spec("fixer");
14428            fixer.command = vec![
14429                "sh".into(),
14430                "-c".into(),
14431                format!("echo started >> '{}'; echo done", marker.display()),
14432            ];
14433            runner.roles.fixer = Some(vec![fixer]);
14434            if graft {
14435                let mut loser = winner.clone();
14436                loser.label = 'B';
14437                loser.index = 1;
14438                runner.state.candidates.push(loser);
14439                runner.state.config.graph.graft_runner_up = true;
14440                runner.state.tally = Some(Tally {
14441                    first_choice: BTreeMap::new(),
14442                    borda: BTreeMap::new(),
14443                    winner: winner.label,
14444                    rankings: 1,
14445                    unanimous_initial: true,
14446                    deliberated: false,
14447                    changed_votes: 0,
14448                    unanimous_final: true,
14449                    tie_break: None,
14450                    judges: 1,
14451                    present: 1,
14452                    quorum: 1,
14453                    met_quorum: true,
14454                    uncontested: None,
14455                });
14456                runner.state.judgements.push(crate::run::Judgement {
14457                    judge: 1,
14458                    seat: "judge-1".into(),
14459                    agent: "judge".into(),
14460                    ranking: vec![winner.label, 'B'],
14461                    reasons: BTreeMap::new(),
14462                    confidence: None,
14463                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14464                    order: vec![0, 1],
14465                    failed: None,
14466                    duration_ms: 0,
14467                });
14468            } else {
14469                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14470                    from_run: "earlier".into(),
14471                    generation: 1,
14472                    mode: crate::recovery::Mode::VerifyRepair,
14473                    reasons: Vec::new(),
14474                    reviewed_head: None,
14475                    rounds: Vec::new(),
14476                    open_findings: Vec::new(),
14477                    evidence: None,
14478                    criteria_inherited: true,
14479                    repair: crate::recovery::Repair::NotStarted,
14480                });
14481            }
14482            let reader = crate::repo_lock::SeatLease::for_checkout(
14483                &winner.worktree,
14484                false,
14485                crate::config::test_coordination().as_ref(),
14486            )
14487            .await
14488            .unwrap();
14489            let pause = runner.pause.clone();
14490            let (result, ()) = tokio::join!(
14491                async {
14492                    if graft {
14493                        runner.graft().await
14494                    } else {
14495                        runner.recovery_repair(&winner).await
14496                    }
14497                },
14498                async {
14499                    tokio::time::sleep(Duration::from_millis(100)).await;
14500                    pause.park();
14501                }
14502            );
14503            result.unwrap();
14504            assert!(!marker.exists(), "a blocked seat must not spawn");
14505            // Exercise the persisted state, as a binary replacement/resume does.
14506            runner.state = RunState::load(&runner.state.id).unwrap();
14507            if graft {
14508                assert_eq!(
14509                    runner.state.graft.as_ref().unwrap().outcome,
14510                    GraftOutcome::WaitingForRestore
14511                );
14512            } else {
14513                assert!(matches!(
14514                    runner.state.recovery.as_ref().unwrap().repair,
14515                    crate::recovery::Repair::WaitingForRestore { .. }
14516                ));
14517            }
14518            drop(reader);
14519            runner.pause = Pause::new();
14520            runner.state.resource_block = None;
14521            // Deferred restoration must still respect an external lock owner.
14522            let external =
14523                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14524            assert_eq!(
14525                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14526                0
14527            );
14528            let leftover = winner.worktree.join("leftover");
14529            std::fs::write(&leftover, "earlier fallback edits").unwrap();
14530            let result = if graft {
14531                runner.graft().await
14532            } else {
14533                runner.recovery_repair(&winner).await
14534            };
14535            assert!(
14536                result
14537                    .unwrap_err()
14538                    .is::<crate::repo_lock::ResourceBlocked>()
14539            );
14540            assert!(leftover.exists());
14541            assert!(!marker.exists());
14542            drop(external);
14543            runner.state.resource_block = None;
14544            let result = if graft {
14545                runner.graft().await
14546            } else {
14547                runner.recovery_repair(&winner).await
14548            };
14549            result.unwrap();
14550            assert!(!leftover.exists(), "restore must precede retry");
14551            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14552            if graft {
14553                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14554            } else {
14555                assert!(matches!(
14556                    runner.state.recovery.as_ref().unwrap().repair,
14557                    crate::recovery::Repair::Done { .. }
14558                ));
14559            }
14560        }
14561    }
14562
14563    #[tokio::test]
14564    #[cfg(unix)]
14565    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
14566        use std::os::fd::AsRawFd as _;
14567        let tmp = tempfile::tempdir().unwrap();
14568        let mut runner = registered_runner(tmp.path()).await;
14569        runner.state.config.graph.timeout_resource_wait = 1;
14570        let winner = runner.state.candidates[0].clone();
14571        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14572        runner.state.tally = Some(Tally {
14573            first_choice: BTreeMap::new(),
14574            borda: BTreeMap::new(),
14575            winner: winner.label,
14576            rankings: 0,
14577            unanimous_initial: false,
14578            deliberated: false,
14579            changed_votes: 0,
14580            unanimous_final: false,
14581            tie_break: None,
14582            judges: 0,
14583            present: 0,
14584            quorum: 0,
14585            met_quorum: true,
14586            uncontested: Some("fixture".into()),
14587        });
14588        runner.state.graft = Some(GraftRecord {
14589            pre: pre.clone(),
14590            post: None,
14591            lines: None,
14592            limit: None,
14593            agent: None,
14594            outcome: GraftOutcome::Running,
14595            requested: Vec::new(),
14596            adoption: None,
14597            adoption_error: None,
14598        });
14599        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
14600        git::commit_all(&winner.worktree, "unfinished graft")
14601            .await
14602            .unwrap();
14603        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14604        let leftover = winner.worktree.join("leftover");
14605        std::fs::write(&leftover, "untrusted edits").unwrap();
14606        let path = runner.state.repo.join(".git/updater.lock");
14607        let external = std::fs::File::create(&path).unwrap();
14608        assert_eq!(
14609            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14610            0
14611        );
14612        assert!(
14613            runner
14614                .graft()
14615                .await
14616                .unwrap_err()
14617                .is::<crate::repo_lock::ResourceBlocked>()
14618        );
14619        assert!(matches!(
14620            runner.state.graft.as_ref().unwrap().outcome,
14621            GraftOutcome::Running
14622        ));
14623        assert_eq!(
14624            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14625            unfinished
14626        );
14627        assert!(leftover.exists());
14628        drop(external);
14629        std::fs::remove_file(&path).unwrap();
14630        std::fs::create_dir(&path).unwrap();
14631        assert!(runner.graft().await.is_err());
14632        assert!(matches!(
14633            runner.state.graft.as_ref().unwrap().outcome,
14634            GraftOutcome::Running
14635        ));
14636        assert_eq!(
14637            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14638            unfinished
14639        );
14640        assert!(leftover.exists());
14641        std::fs::remove_dir(&path).unwrap();
14642        runner.graft().await.unwrap();
14643        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14644        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
14645        assert!(!leftover.exists());
14646    }
14647
14648    #[tokio::test]
14649    #[cfg(unix)]
14650    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
14651        use std::os::fd::AsRawFd as _;
14652        let tmp = tempfile::tempdir().unwrap();
14653        let mut runner = registered_runner(tmp.path()).await;
14654        runner.state.prep_complete = Some(false);
14655        runner.state.config.graph.timeout_resource_wait = 1;
14656        runner.state.save().unwrap();
14657        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14658        assert_eq!(
14659            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14660            0
14661        );
14662        runner.execute().await.unwrap();
14663        let disk = RunState::load(&runner.state.id).unwrap();
14664        assert!(disk.driver_exited);
14665        assert_eq!(disk.driver_pid, None);
14666        assert_eq!(runner.state.status, RunStatus::Blocked);
14667        assert_eq!(
14668            runner.state.resource_block.as_ref().unwrap().kind,
14669            "repository coordination"
14670        );
14671        assert!(runner.state.prep_failure.is_none());
14672        assert!(runner.state.seats.is_empty());
14673        runner.pause.park();
14674        // Include headroom for Git subprocesses under parallel test load;
14675        // the repository contention deadline itself remains one second.
14676        tokio::time::timeout(Duration::from_secs(10), runner.execute())
14677            .await
14678            .unwrap()
14679            .unwrap();
14680        assert!(runner.state.parked);
14681        assert!(runner.state.prep_failure.is_none());
14682        let disk = RunState::load(&runner.state.id).unwrap();
14683        assert!(disk.driver_exited);
14684        assert_eq!(disk.driver_pid, None);
14685    }
14686
14687    #[tokio::test]
14688    #[cfg(unix)]
14689    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
14690        use std::os::fd::AsRawFd as _;
14691        let tmp = tempfile::tempdir().unwrap();
14692        let mut runner = registered_runner(tmp.path()).await;
14693        runner.state.prep_complete = Some(false);
14694        runner.state.config.graph.timeout_resource_wait = 30;
14695        runner.state.save().unwrap();
14696        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14697        assert_eq!(
14698            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14699            0
14700        );
14701        assert!(
14702            tokio::time::timeout(Duration::from_millis(100), runner.execute())
14703                .await
14704                .is_err()
14705        );
14706        let disk = RunState::load(&runner.state.id).unwrap();
14707        assert!(disk.driver_exited);
14708        assert_eq!(disk.driver_pid, None);
14709        assert!(disk.status.resumable());
14710        assert!(runner.state.driver_exited);
14711        assert_eq!(runner.state.driver_pid, None);
14712        runner.state.save().unwrap();
14713        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
14714    }
14715
14716    #[tokio::test]
14717    #[cfg(unix)]
14718    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
14719        let tmp = tempfile::tempdir().unwrap();
14720        let mut runner = registered_runner(tmp.path()).await;
14721        let winner = runner.state.candidates[0].clone();
14722        let cache = tmp.path().join("cache");
14723        let started = tmp.path().join("pre-gate-started");
14724        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
14725        runner.state.config.verify.gate =
14726            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
14727        let home = crate::run::home();
14728        let owner =
14729            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
14730        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
14731            crate::cache::AcquireOutcome::Acquired(g) => g,
14732            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
14733        };
14734        let mut gate = Box::pin(runner.run_gate(&winner));
14735        assert!(
14736            tokio::time::timeout(Duration::from_millis(150), &mut gate)
14737                .await
14738                .is_err()
14739        );
14740        assert!(!started.exists(), "pre-gate must wait for the cache too");
14741        // Simulate the wave that already owns the cache entering invoke.
14742        // It must obtain the editor reservation and finish without waiting
14743        // for the gate that is queued behind its cache.
14744        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
14745        let editor = crate::repo_lock::SeatLease::checkout_with(
14746            &winner.worktree,
14747            true,
14748            crate::config::test_coordination().as_ref(),
14749            &wait,
14750        )
14751        .await
14752        .unwrap();
14753        drop(editor);
14754        guard.release();
14755        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14756            .await
14757            .unwrap()
14758            .unwrap();
14759        drop(gate);
14760        assert!(outcomes.iter().all(CommandOutcome::ok));
14761        assert!(started.exists());
14762    }
14763
14764    #[tokio::test]
14765    #[cfg(unix)]
14766    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
14767        use std::os::fd::AsRawFd as _;
14768
14769        ask_test_home();
14770        let tmp = tempfile::tempdir().unwrap();
14771        let repo = tmp.path().join("repo");
14772        std::fs::create_dir_all(&repo).unwrap();
14773        init_repo(&repo);
14774        let common = repo.join(".git");
14775        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14776        let mut runner = runner_at(RunStatus::Prep);
14777        runner.state.repo = repo.clone();
14778        runner.state.config.coordination = crate::config::test_coordination();
14779        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14780        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14781        runner.state.config.graph.advise = false;
14782        runner.roles.implementers = vec![spec("implement")];
14783        runner.roles.judges = vec![spec("judge")];
14784        runner.prep().await.unwrap();
14785        let winner = runner.state.candidates[0].clone();
14786        // Emulate edits already left by a completed seat. The fixture CLI
14787        // only reports completion; it performs no repository operations.
14788        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
14789        let completed = tmp.path().join("seat-completed");
14790        runner.roles.implementers[0].command = vec![
14791            "sh".into(),
14792            "-c".into(),
14793            format!("touch '{}'; echo completed", completed.display()),
14794        ];
14795        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
14796        assert_eq!(
14797            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14798            0
14799        );
14800        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14801        let mut implement = Box::pin(runner.implement());
14802        tokio::select! {
14803            result = &mut implement => panic!("rescue must wait: {result:?}"),
14804            result = tokio::time::timeout(Duration::from_secs(10), async {
14805                while !completed.exists() {
14806                    tokio::time::sleep(Duration::from_millis(20)).await;
14807                }
14808            }) => result.unwrap(),
14809        }
14810        assert!(
14811            tokio::time::timeout(Duration::from_millis(200), &mut implement)
14812                .await
14813                .is_err()
14814        );
14815        assert_eq!(
14816            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14817            before
14818        );
14819        assert_eq!(
14820            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14821            0
14822        );
14823        tokio::time::timeout(Duration::from_secs(10), &mut implement)
14824            .await
14825            .unwrap()
14826            .unwrap();
14827        drop(implement);
14828        assert_ne!(
14829            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14830            before
14831        );
14832        assert!(git::is_clean(&winner.worktree).await.unwrap());
14833
14834        let started = tmp.path().join("pre-gate-started");
14835        let proceed = tmp.path().join("pre-gate-proceed");
14836        let checked = tmp.path().join("gate-ran");
14837        runner.state.config.verify.pre_gate = vec![format!(
14838            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14839            started.display(),
14840            proceed.display()
14841        )];
14842        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
14843        assert_eq!(
14844            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14845            0
14846        );
14847        let mut gate = Box::pin(runner.run_gate(&winner));
14848        assert!(
14849            tokio::time::timeout(Duration::from_millis(200), &mut gate)
14850                .await
14851                .is_err()
14852        );
14853        assert!(!started.exists());
14854        assert!(!checked.exists());
14855        assert!(git::is_clean(&winner.worktree).await.unwrap());
14856        assert_eq!(
14857            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14858            0
14859        );
14860        tokio::select! {
14861            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
14862            result = tokio::time::timeout(Duration::from_secs(10), async {
14863                while !started.exists() {
14864                    tokio::time::sleep(Duration::from_millis(20)).await;
14865                }
14866            }) => result.unwrap(),
14867        }
14868        assert_eq!(
14869            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14870            -1,
14871            "mechanical edits must hold the real lock"
14872        );
14873        std::fs::write(proceed, "ready").unwrap();
14874        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14875            .await
14876            .unwrap()
14877            .unwrap();
14878        drop(gate);
14879        assert!(outcomes.iter().all(CommandOutcome::ok));
14880        assert!(checked.exists());
14881        assert!(runner.state.pre_gate_commit.is_some());
14882        assert!(git::is_clean(&winner.worktree).await.unwrap());
14883        assert_eq!(
14884            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14885            0,
14886            "gate completion must release coordination"
14887        );
14888        drop(outsider);
14889        std::fs::remove_file(common.join("updater.lock")).unwrap();
14890        std::fs::create_dir(common.join("updater.lock")).unwrap();
14891        std::fs::remove_file(&started).unwrap();
14892        std::fs::remove_file(&checked).unwrap();
14893        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14894        let error = runner.run_gate(&winner).await.unwrap_err();
14895        assert!(
14896            error
14897                .to_string()
14898                .contains("acquire mandatory repository lock")
14899        );
14900        assert!(
14901            !started.exists(),
14902            "failed acquisition must prevent pre-gate"
14903        );
14904        assert!(!checked.exists(), "failed acquisition must prevent gate");
14905        assert!(git::is_clean(&winner.worktree).await.unwrap());
14906        assert_eq!(
14907            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14908            head
14909        );
14910    }
14911
14912    #[tokio::test]
14913    #[cfg(unix)]
14914    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
14915        use std::os::fd::AsRawFd as _;
14916        use std::time::Duration;
14917
14918        ask_test_home();
14919        let tmp = tempfile::tempdir().unwrap();
14920        let repo = tmp.path().join("repo");
14921        std::fs::create_dir_all(&repo).unwrap();
14922        init_repo(&repo);
14923        let common = repo.join(".git");
14924        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14925        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
14926        assert_eq!(
14927            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14928            0
14929        );
14930        let mut runner = runner_at(RunStatus::Prep);
14931        runner.state.repo = repo.clone();
14932        runner.state.config.coordination = crate::config::test_coordination();
14933        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14934        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14935        runner.state.config.graph.advise = true;
14936        runner.state.config.graph.advisors = 1;
14937        runner.roles.implementers = vec![spec("implement")];
14938        runner.roles.judges = vec![spec("judge")];
14939        let root = runner.state.worktree_root();
14940        let started = tmp.path().join("setup-started");
14941        let proceed = tmp.path().join("setup-proceed");
14942        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14943            run: Some(format!(
14944                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14945                started.display(),
14946                proceed.display()
14947            )),
14948            ..Default::default()
14949        }];
14950        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
14951        let config = git::git(&repo, &["config", "--local", "--list"])
14952            .await
14953            .unwrap();
14954        let mut prep = Box::pin(runner.prep());
14955        assert!(
14956            tokio::time::timeout(Duration::from_millis(200), &mut prep)
14957                .await
14958                .is_err()
14959        );
14960        assert!(
14961            !root.exists(),
14962            "contention must prevent even checkout creation"
14963        );
14964        assert!(!started.exists(), "contention must prevent setup");
14965        assert_eq!(
14966            git::git(&repo, &["branch", "--list"]).await.unwrap(),
14967            branches
14968        );
14969        assert_eq!(
14970            git::git(&repo, &["config", "--local", "--list"])
14971                .await
14972                .unwrap(),
14973            config
14974        );
14975        drop(lock);
14976        tokio::select! {
14977            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
14978            result = tokio::time::timeout(Duration::from_secs(10), async {
14979                while !started.exists() {
14980                    tokio::time::sleep(Duration::from_millis(20)).await;
14981                }
14982            }) => result.unwrap(),
14983        }
14984        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
14985        assert_eq!(
14986            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14987            -1
14988        );
14989        assert!(
14990            tokio::time::timeout(
14991                Duration::from_millis(100),
14992                crate::repo_lock::SeatLease::for_checkout(
14993                    &repo,
14994                    true,
14995                    crate::config::test_coordination().as_ref()
14996                )
14997            )
14998            .await
14999            .is_err(),
15000            "preparation must also exclude local editors"
15001        );
15002        std::fs::write(proceed, "ready").unwrap();
15003        tokio::time::timeout(Duration::from_secs(10), &mut prep)
15004            .await
15005            .unwrap()
15006            .unwrap();
15007        drop(prep);
15008        assert_eq!(runner.state.prep_complete, Some(true));
15009        assert!(root.join("judge-1").exists());
15010        assert!(root.join("advisor-1").exists());
15011        assert_eq!(
15012            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15013            0
15014        );
15015    }
15016
15017    #[tokio::test]
15018    #[cfg(unix)]
15019    async fn preparation_lock_failure_prevents_repository_changes() {
15020        ask_test_home();
15021        let tmp = tempfile::tempdir().unwrap();
15022        let repo = tmp.path().join("repo");
15023        std::fs::create_dir_all(&repo).unwrap();
15024        init_repo(&repo);
15025        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15026        // A directory at the required lock path makes opening the real lock
15027        // fail, rather than merely exercising an invalid registration.
15028        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15029        let mut runner = runner_at(RunStatus::Prep);
15030        runner.state.repo = repo.clone();
15031        runner.state.config.coordination = crate::config::test_coordination();
15032        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15033        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15034        runner.roles.implementers = vec![spec("implement")];
15035        let error = runner.prep().await.unwrap_err();
15036        assert!(
15037            error
15038                .to_string()
15039                .contains("acquire mandatory repository lock")
15040        );
15041        assert!(!runner.state.worktree_root().exists());
15042        assert!(runner.state.candidates.is_empty());
15043        assert!(!repo.join(".git/config.worktree").exists());
15044        assert_eq!(
15045            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15046            "* main"
15047        );
15048    }
15049
15050    #[tokio::test]
15051    async fn preparation_records_creation_intent_before_git_can_fail() {
15052        ask_test_home();
15053        let tmp = tempfile::tempdir().unwrap();
15054        let repo = tmp.path().join("repo");
15055        std::fs::create_dir_all(&repo).unwrap();
15056        init_repo(&repo);
15057        let mut runner = runner_at(RunStatus::Prep);
15058        runner.state.repo = repo.clone();
15059        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15060        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15061        runner.state.config.blind.commit_msg_hook = false;
15062        runner.roles.implementers = vec![spec("one")];
15063        let label = blind::assign_labels(1, runner.state.seed)[0];
15064        let branch = runner.state.branch_for(label);
15065        // A conflicting branch makes Git refuse creation before a candidate
15066        // can be registered. The durable intent must still identify it.
15067        git::git(&repo, &["branch", &branch]).await.unwrap();
15068        assert!(runner.prep().await.is_err());
15069        let saved = RunState::load(&runner.state.id).unwrap();
15070        assert!(saved.candidates.is_empty());
15071        let event = saved
15072            .events
15073            .iter()
15074            .find(|e| e.node == "prep-intent")
15075            .unwrap();
15076        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15077        assert_eq!(intent["branch"], branch);
15078        assert_eq!(intent["start"], runner.state.base_commit);
15079        assert_eq!(
15080            intent["worktree"],
15081            runner
15082                .state
15083                .worktree_root()
15084                .join(format!("cand-{label}"))
15085                .to_string_lossy()
15086                .as_ref()
15087        );
15088        assert_eq!(intent["picks"], serde_json::json!([]));
15089    }
15090
15091    #[tokio::test]
15092    async fn invalid_pinned_sources_create_no_partial_worktrees() {
15093        ask_test_home();
15094        let tmp = tempfile::tempdir().unwrap();
15095        let repo = tmp.path().join("repo");
15096        std::fs::create_dir_all(&repo).unwrap();
15097        init_repo(&repo);
15098        let mut runner = runner_at(RunStatus::Prep);
15099        runner.state.repo = repo.clone();
15100        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15101        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15102        runner.state.sources.pick_commits.push(refs::Input {
15103            token: "missing".into(),
15104            sha: "0".repeat(40),
15105            provenance: refs::Provenance::ExplicitCli,
15106        });
15107        assert!(runner.prep().await.is_err());
15108        assert!(runner.state.candidates.is_empty());
15109        assert!(!runner.state.worktree_root().exists());
15110    }
15111
15112    #[tokio::test]
15113    async fn prose_and_legacy_inference_never_import_into_candidates() {
15114        ask_test_home();
15115        let tmp = tempfile::tempdir().unwrap();
15116        let repo = tmp.path().join("repo");
15117        std::fs::create_dir_all(&repo).unwrap();
15118        init_repo(&repo);
15119        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15120        git::git(&repo, &["checkout", "-b", "magi/history/A"])
15121            .await
15122            .unwrap();
15123        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15124        git::git(&repo, &["add", "."]).await.unwrap();
15125        git::git(&repo, &["commit", "-m", "historical implementation"])
15126            .await
15127            .unwrap();
15128        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15129        git::git(&repo, &["checkout", "main"]).await.unwrap();
15130        for instruction in [
15131            format!("Example: magi/history/A at {source}"),
15132            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15133            format!("Never import magi/history/A or {source}"),
15134            format!("Conversation answer: I mentioned {source} only as background"),
15135        ] {
15136            let mut runner = runner_at(RunStatus::Prep);
15137            runner.state.repo = repo.clone();
15138            runner.state.base_commit = base.clone();
15139            runner.state.instruction = instruction.clone();
15140            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15141            runner.state.config.blind.commit_msg_hook = false;
15142            runner.state.seeds = vec![refs::Seed {
15143                token: "magi/history/A".into(),
15144                sha: source.clone(),
15145                kind: refs::SeedKind::Unmerged,
15146                branch: true,
15147                detail: String::new(),
15148            }];
15149            runner.roles.implementers = vec![spec("one")];
15150            assert_eq!(seeded_instruction(&runner.state), instruction);
15151            // Model an old record with inferred seeds but no structured input field.
15152            let mut legacy = serde_json::to_value(&runner.state).unwrap();
15153            legacy.as_object_mut().unwrap().remove("sources");
15154            runner.state = serde_json::from_value(legacy).unwrap();
15155            assert_eq!(runner.state.sources, refs::Sources::default());
15156            runner.state.save().unwrap();
15157            runner.state = RunState::load(&runner.state.id).unwrap();
15158            runner.prep().await.unwrap();
15159            for candidate in &runner.state.candidates {
15160                assert_eq!(
15161                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15162                    base
15163                );
15164                assert!(!candidate.worktree.join("unwanted").exists());
15165            }
15166        }
15167    }
15168
15169    #[tokio::test]
15170    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15171        ask_test_home();
15172        let tmp = tempfile::tempdir().unwrap();
15173        let repo = tmp.path().join("repo");
15174        std::fs::create_dir_all(&repo).unwrap();
15175        init_repo(&repo);
15176        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15177        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15178        std::fs::write(repo.join("referenced"), "required commit").unwrap();
15179        git::git(&repo, &["add", "."]).await.unwrap();
15180        git::git(&repo, &["commit", "-m", "required seed"])
15181            .await
15182            .unwrap();
15183        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15184        git::git(&repo, &["checkout", "main"]).await.unwrap();
15185        let mut runner = runner_at(RunStatus::Prep);
15186        runner.state.repo = repo.clone();
15187        runner.state.base_commit = base.clone();
15188        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15189        runner.state.config.graph.advise = false;
15190        runner.state.config.blind.commit_msg_hook = false;
15191        runner.state.sources =
15192            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15193                .await
15194                .unwrap();
15195        runner.roles.implementers = vec![spec("one"), spec("two")];
15196        runner.roles.judges = vec![spec("judge")];
15197        let label = blind::assign_labels(2, runner.state.seed)[0];
15198        let branch = runner.state.branch_for(label);
15199        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15200        git::worktree_add_branch(&repo, &worktree, &branch, &base)
15201            .await
15202            .unwrap();
15203        runner.state.candidates.push(Candidate {
15204            index: 0,
15205            label,
15206            agent: "one".into(),
15207            branch,
15208            worktree,
15209            summary: String::new(),
15210            stat: String::new(),
15211            files: 0,
15212            commits: 0,
15213            empty: false,
15214            failed: None,
15215            verified_noop: None,
15216            duration_ms: 0,
15217            folded: false,
15218        });
15219        runner.state.save().unwrap();
15220        // Reload the checkpoint immediately before its first cherry-pick.
15221        runner.state = RunState::load(&runner.state.id).unwrap();
15222        assert_eq!(runner.state.prep_complete, Some(false));
15223        runner.prep().await.unwrap();
15224        assert_eq!(runner.state.prep_complete, Some(true));
15225        assert_eq!(runner.state.candidates.len(), 2);
15226        for candidate in &runner.state.candidates {
15227            assert_eq!(
15228                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15229                git::tree_of(&repo, &seed).await.unwrap()
15230            );
15231            assert_eq!(
15232                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15233                "required commit"
15234            );
15235        }
15236        assert!(runner.state.worktree_root().join("judge-1").exists());
15237        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15238        runner.prep().await.unwrap();
15239        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15240
15241        runner.state.prep_complete = Some(false);
15242        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15243            run: Some(format!(
15244                "test -f '{}'",
15245                tmp.path().join("dependency-ready").display()
15246            )),
15247            ..Default::default()
15248        }];
15249        assert!(runner.prep().await.is_err());
15250        assert!(runner.state.prep_failure.is_some());
15251        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15252        runner.state = RunState::load(&runner.state.id).unwrap();
15253        assert!(
15254            runner.prep().await.is_err(),
15255            "correcting the cause alone must not auto-retry"
15256        );
15257        let failed_at = runner.state.events.last().unwrap().at;
15258        assert!(
15259            runner.retry_preparation(failed_at).is_err(),
15260            "old approvals cannot unlock the hold"
15261        );
15262        let mut task = crate::queue::Task::new(
15263            "retry".into(),
15264            "retry setup".into(),
15265            repo.clone(),
15266            crate::queue::Source::Human,
15267        );
15268        task.start(runner.state.id.clone());
15269        task.hold_machine(Some("setup failed".into()));
15270        task.release_operator("cli", None);
15271        runner
15272            .retry_preparation(task.resumption.as_ref().unwrap().at)
15273            .unwrap();
15274        runner.prep().await.unwrap();
15275        assert!(runner.state.prep_failure.is_none());
15276        assert_eq!(runner.state.prep_complete, Some(true));
15277        assert_eq!(runner.state.candidates.len(), 2);
15278        assert!(runner.state.worktree_root().join("judge-1").exists());
15279        for candidate in &runner.state.candidates {
15280            assert_eq!(
15281                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15282                git::tree_of(&repo, &seed).await.unwrap()
15283            );
15284        }
15285    }
15286
15287    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15288        let mut heads = Vec::new();
15289        for candidate in candidates {
15290            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15291        }
15292        heads
15293    }
15294
15295    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15296        ask_test_home();
15297        let tmp = tempfile::tempdir().unwrap();
15298        let repo = tmp.path().join("repo");
15299        std::fs::create_dir(&repo).unwrap();
15300        init_repo(&repo);
15301        let mut runner = runner_at(RunStatus::Reviewing);
15302        runner.state.repo = repo.clone();
15303        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15304        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15305        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15306            let branch = format!("candidate-{label}");
15307            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15308            git::worktree_add_branch(&repo, &worktree, &branch, "main")
15309                .await
15310                .unwrap();
15311            if label == 'B' {
15312                std::fs::write(worktree.join("change"), "winner").unwrap();
15313                git::rescue_commit(&worktree, "candidate change")
15314                    .await
15315                    .unwrap();
15316            }
15317            runner.state.candidates.push(Candidate {
15318                index,
15319                label,
15320                agent: "alpha".into(),
15321                branch,
15322                worktree,
15323                summary: "completed reply".into(),
15324                stat: String::new(),
15325                files: usize::from(label == 'B'),
15326                commits: usize::from(label == 'B'),
15327                empty: label == 'A',
15328                failed: None,
15329                verified_noop: None,
15330                duration_ms: 1,
15331                folded: false,
15332            });
15333        }
15334        runner.state.judge_skipped = true;
15335        runner.state.tally = Some(Tally {
15336            first_choice: BTreeMap::new(),
15337            borda: BTreeMap::new(),
15338            winner: 'B',
15339            rankings: 0,
15340            unanimous_initial: false,
15341            deliberated: false,
15342            changed_votes: 0,
15343            unanimous_final: false,
15344            tie_break: None,
15345            judges: 0,
15346            present: 0,
15347            quorum: 0,
15348            met_quorum: true,
15349            uncontested: Some("only B produced a change".into()),
15350        });
15351        runner.state.save().unwrap();
15352        (tmp, runner)
15353    }
15354
15355    #[tokio::test]
15356    #[cfg(unix)]
15357    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15358        for (solo, failed_checkpoint) in
15359            [(true, false), (false, false), (true, true), (false, true)]
15360        {
15361            let (_tmp, mut runner) = preservation_resume_fixture().await;
15362            if solo {
15363                runner.state.candidates.retain(|c| c.label == 'A');
15364            }
15365            runner.state.candidates[0].empty = false;
15366            runner.state.candidates[0].summary.clear();
15367            runner.state.tally = None;
15368            runner.state.judge_skipped = false;
15369            runner.state.status = RunStatus::Implementing;
15370            runner.state.config.disk.min_free_bytes = u64::MAX;
15371            let mut implementer = spec("alpha");
15372            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15373            runner.roles.implementers = vec![implementer];
15374            let patch = runner.state.dir().join("artifacts/cand-A.patch");
15375            if failed_checkpoint {
15376                std::fs::create_dir_all(&patch).unwrap();
15377            }
15378            runner.state.save().unwrap();
15379            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15380            runner.state = RunState::load(&runner.state.id).unwrap();
15381            assert_eq!(runner.state.status, RunStatus::Blocked);
15382            assert!(runner.state.interrupted_implementations.contains(&'A'));
15383            assert_eq!(runner.state.candidates[0].commits, 0);
15384            assert!(!runner.state.candidates[0].empty);
15385            assert!(runner.state.candidates[0].failed.is_none());
15386            if failed_checkpoint {
15387                assert!(runner.state.candidate_preservation.contains_key(&'A'));
15388                std::fs::remove_dir(patch).unwrap();
15389            }
15390            runner.state.resource_block = None;
15391            runner.state.config.disk.min_free_bytes = 0;
15392            runner.roles.implementers[0].command = vec![
15393                "sh".into(),
15394                "-c".into(),
15395                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15396            ];
15397            runner.implement().await.unwrap();
15398            assert!(runner.state.interrupted_implementations.is_empty());
15399            assert!(runner.state.candidate_preservation.is_empty());
15400            assert_eq!(runner.state.candidates[0].commits, 1);
15401            assert!(!runner.state.candidates[0].empty);
15402            assert!(
15403                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15404                    .unwrap()
15405                    .contains("retried empty interrupted seat")
15406            );
15407            if !solo {
15408                assert_eq!(runner.state.candidates[1].commits, 1);
15409            }
15410        }
15411    }
15412
15413    #[tokio::test]
15414    #[cfg(unix)]
15415    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15416        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15417            let (_tmp, mut runner) = preservation_resume_fixture().await;
15418            runner.state.candidates.retain(|c| c.label == 'A');
15419            runner.state.candidates[0].empty = false;
15420            runner.state.candidates[0].summary.clear();
15421            runner.state.tally = None;
15422            runner.state.judge_skipped = false;
15423            runner.state.status = RunStatus::Implementing;
15424            // Force the real post-spawn watchdog, independently of available
15425            // disk space. Dirty/committed work predates its first measurement.
15426            runner.state.config.disk.min_free_bytes = u64::MAX;
15427            let wt = runner.state.candidates[0].worktree.clone();
15428            std::fs::write(
15429                wt.join("interrupted-source"),
15430                "retain implementation work\n",
15431            )
15432            .unwrap();
15433            if committed {
15434                git::rescue_commit(&wt, "implementation checkpoint")
15435                    .await
15436                    .unwrap();
15437            }
15438            let lock = git::git(
15439                &wt,
15440                &[
15441                    "rev-parse",
15442                    "--path-format=absolute",
15443                    "--git-path",
15444                    "index.lock",
15445                ],
15446            )
15447            .await
15448            .unwrap();
15449            if block_rescue {
15450                std::fs::write(&lock, "fixture-owned lock").unwrap();
15451            }
15452            let mut implementer = spec("alpha");
15453            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15454            runner.roles.implementers = vec![implementer];
15455            runner.state.save().unwrap();
15456            let result = runner.implement().await;
15457            assert_eq!(result.is_err(), block_rescue);
15458            assert_eq!(
15459                runner.state.resource_block.as_ref().unwrap().kind,
15460                "disk pressure"
15461            );
15462            assert!(
15463                runner
15464                    .state
15465                    .events
15466                    .iter()
15467                    .any(|e| e.message.contains("agent execution interrupted"))
15468            );
15469            assert!(
15470                runner
15471                    .state
15472                    .dir()
15473                    .join("artifacts/candidate-A-recovery/staged.patch")
15474                    .exists()
15475            );
15476            if block_rescue {
15477                let saved = RunState::load(&runner.state.id).unwrap();
15478                assert!(saved.candidate_preservation.contains_key(&'A'));
15479                assert!(!saved.candidates[0].empty);
15480                assert_eq!(
15481                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15482                    "retain implementation work\n"
15483                );
15484                std::fs::remove_file(lock).unwrap();
15485                runner.state = saved;
15486            } else {
15487                assert_eq!(runner.state.status, RunStatus::Blocked);
15488                assert_eq!(runner.state.candidates[0].commits, 1);
15489            }
15490            // Recovery must save the candidate before deciding whether another
15491            // implementation is needed. No implementer is available on retry.
15492            runner.roles.implementers.clear();
15493            runner.state.config.disk.min_free_bytes = 0;
15494            runner.state.resource_block = None;
15495            runner.implement().await.unwrap();
15496            assert!(runner.state.candidate_preservation.is_empty());
15497            assert_eq!(runner.state.candidates[0].commits, 1);
15498            assert!(!runner.state.candidates[0].empty);
15499            assert!(git::is_clean(&wt).await.unwrap());
15500            assert!(
15501                crate::run::read_artifact(&runner.state, "cand-A.patch")
15502                    .unwrap()
15503                    .contains("retain implementation work")
15504            );
15505        }
15506    }
15507
15508    #[tokio::test]
15509    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15510        let (_tmp, mut runner) = preservation_resume_fixture().await;
15511        // Schema 15 already confirmed the unfolded no-change result.
15512        runner.implement().await.unwrap();
15513        assert!(runner.state.candidate_preservation.is_empty());
15514        assert!(
15515            !runner
15516                .state
15517                .dir()
15518                .join("artifacts/candidate-A-recovery")
15519                .exists()
15520        );
15521        runner.fold_losers().await.unwrap();
15522        assert!(!runner.state.candidates[0].worktree.exists());
15523        for schema in [14, crate::run::SCHEMA] {
15524            runner.state.schema = schema;
15525            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15526            runner.state.save().unwrap();
15527            runner.state = RunState::load(&runner.state.id).unwrap();
15528            runner.implement().await.unwrap();
15529            assert!(runner.state.candidate_preservation.is_empty());
15530            assert!(runner.state.candidates[0].folded);
15531            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15532        }
15533    }
15534
15535    #[tokio::test]
15536    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15537        let (_tmp, mut runner) = preservation_resume_fixture().await;
15538        runner.state.candidates.retain(|c| c.label == 'A');
15539        runner.state.tally = None;
15540        runner.state.judge_skipped = false;
15541        let candidate = &mut runner.state.candidates[0];
15542        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15543        candidate.failed = Some("deployment validation unavailable".into());
15544        std::fs::write(
15545            candidate.worktree.join("repair"),
15546            "future launcher policy\n",
15547        )
15548        .unwrap();
15549        runner
15550            .state
15551            .candidate_preservation
15552            .insert('A', "awaiting persistence".into());
15553        runner.persist_candidates().await.unwrap();
15554        let candidate = &runner.state.candidates[0];
15555        assert!(!candidate.empty);
15556        assert_eq!(candidate.commits, 1);
15557        assert_eq!(candidate.files, 1);
15558        assert!(candidate.failed.is_none());
15559        assert!(candidate.summary.contains("EPERM"));
15560        assert!(
15561            crate::run::read_artifact(&runner.state, "cand-A.patch")
15562                .unwrap()
15563                .contains("future launcher policy")
15564        );
15565        assert!(runner.state.candidate_preservation.is_empty());
15566    }
15567
15568    #[tokio::test]
15569    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
15570        let (_tmp, mut runner) = preservation_resume_fixture().await;
15571        let wt = runner.state.candidates[0].worktree.clone();
15572        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
15573        runner.state.schema = 14;
15574        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15575        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15576        let tally = serde_json::to_value(&runner.state.tally).unwrap();
15577        for _ in 0..2 {
15578            assert!(runner.implement().await.is_err());
15579            assert_eq!(runner.state.status, RunStatus::Blocked);
15580            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15581            assert!(
15582                runner.state.candidates[0].empty,
15583                "old verdict must remain unusable"
15584            );
15585            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
15586            assert!(runner.state.judge_skipped);
15587            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
15588            assert!(
15589                crate::run::read_artifact(&runner.state, "cand-A.patch")
15590                    .unwrap()
15591                    .contains("owned edits")
15592            );
15593            assert!(wt.exists());
15594            assert_eq!(
15595                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
15596                    .await
15597                    .unwrap(),
15598                1
15599            );
15600            runner.state = RunState::load(&runner.state.id).unwrap();
15601        }
15602    }
15603
15604    #[tokio::test]
15605    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
15606        let (_tmp, mut runner) = preservation_resume_fixture().await;
15607        let wt = runner.state.candidates[0].worktree.clone();
15608        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
15609        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
15610        let index_lock = PathBuf::from(
15611            git::git(
15612                &wt,
15613                &[
15614                    "rev-parse",
15615                    "--path-format=absolute",
15616                    "--git-path",
15617                    "index.lock",
15618                ],
15619            )
15620            .await
15621            .unwrap(),
15622        );
15623        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15624        runner.state.schema = 14;
15625        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15626        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15627        assert!(runner.implement().await.is_err());
15628        assert!(runner.after_implement().is_err());
15629        assert!(runner.fold_losers().await.is_err());
15630        assert_eq!(
15631            std::fs::read_to_string(wt.join("recovered")).unwrap(),
15632            "failed seat edits"
15633        );
15634        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15635        runner.state = RunState::load(&runner.state.id).unwrap();
15636        std::fs::remove_file(index_lock).unwrap();
15637        // Recovery finds changes that the old uncontested result never judged.
15638        assert!(runner.implement().await.is_err());
15639        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15640        assert!(runner.state.candidates[0].empty);
15641        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15642        assert!(
15643            crate::run::read_artifact(&runner.state, "cand-A.patch")
15644                .unwrap()
15645                .contains("failed seat edits")
15646        );
15647        assert!(wt.exists());
15648    }
15649
15650    #[tokio::test]
15651    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
15652        let (_tmp, mut runner) = preservation_resume_fixture().await;
15653        let repo = runner.state.repo.clone();
15654        let common = git::git(
15655            &repo,
15656            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
15657        )
15658        .await
15659        .unwrap();
15660        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
15661        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
15662        assert!(runner.fold_losers().await.is_err());
15663        assert!(!runner.state.candidates[0].worktree.exists());
15664        assert!(!runner.state.candidates[0].folded);
15665        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
15666        runner.state = RunState::load(&runner.state.id).unwrap();
15667        std::fs::remove_file(branch_lock).unwrap();
15668        let judge = runner.state.worktree_root().join("judge-1");
15669        git::worktree_add_detached(&repo, &judge, "main")
15670            .await
15671            .unwrap();
15672        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
15673        runner.roles.judges = vec![spec("judge")];
15674        assert!(runner.fold_losers().await.is_err());
15675        runner.state = RunState::load(&runner.state.id).unwrap();
15676        assert!(
15677            runner.state.candidates[0].folded,
15678            "candidate progress saved before judge failure"
15679        );
15680        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
15681        assert!(judge.join("uncommitted").exists());
15682        std::fs::remove_file(judge.join("uncommitted")).unwrap();
15683        runner.fold_losers().await.unwrap();
15684        assert!(!judge.exists());
15685        runner.fold_losers().await.unwrap();
15686    }
15687
15688    #[tokio::test]
15689    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
15690        let (_tmp, mut runner) = preservation_resume_fixture().await;
15691        runner.state.candidates.retain(|c| c.label == 'B');
15692        runner.state.tally = None;
15693        runner.state.judge_skipped = false;
15694        runner.state.prep_complete = Some(true);
15695        runner.state.config.graph.advise = false;
15696        runner.state.config.graph.selected_implementer = Some("alpha".into());
15697        let wt = runner.state.candidates[0].worktree.clone();
15698        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
15699        runner
15700            .state
15701            .candidate_preservation
15702            .insert('B', "awaiting persistence".into());
15703        let index_lock = git::git(
15704            &wt,
15705            &[
15706                "rev-parse",
15707                "--path-format=absolute",
15708                "--git-path",
15709                "index.lock",
15710            ],
15711        )
15712        .await
15713        .unwrap();
15714        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15715        runner.state.save().unwrap();
15716        assert!(runner.execute_graph().await.is_err());
15717        assert_eq!(runner.state.status, RunStatus::Blocked);
15718        assert!(runner.state.candidate_preservation.contains_key(&'B'));
15719        std::fs::remove_file(index_lock).unwrap();
15720        // Park at the next graph boundary: this tests the real selected-seat
15721        // hold guard without dispatching reviewers or publishing anything.
15722        for _ in 0..2 {
15723            runner.state = RunState::load(&runner.state.id).unwrap();
15724            runner.pause.park();
15725            runner.execute_graph().await.unwrap();
15726            assert_eq!(runner.state.status, RunStatus::Implementing);
15727            assert!(runner.state.candidate_preservation.is_empty());
15728            assert_eq!(runner.state.candidates[0].commits, 2);
15729            assert!(runner.state.tally.is_none());
15730        }
15731        runner.implement().await.unwrap();
15732        assert_eq!(runner.state.status, RunStatus::Judging);
15733        // A genuine selected-implementer hold must still require release.
15734        runner.state.status = RunStatus::Blocked;
15735        runner.state.candidates[0].failed = Some("implementation failed".into());
15736        runner.state.save().unwrap();
15737        let error = runner.execute_graph().await.unwrap_err();
15738        assert!(
15739            error
15740                .to_string()
15741                .contains("selected implementation is held")
15742        );
15743    }
15744
15745    #[tokio::test]
15746    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
15747        let (_tmp, mut runner) = preservation_resume_fixture().await;
15748        let repo = runner.state.repo.clone();
15749        let candidate = runner.state.candidates[0].clone();
15750        // Remove only the disposable fixture directory, leaving Git's linked
15751        // worktree registration and branch intact, as in a vanished checkout.
15752        let registered = git::worktree_holding(&repo, &candidate.branch)
15753            .await
15754            .unwrap()
15755            .unwrap();
15756        assert_eq!(
15757            registered.canonicalize().unwrap(),
15758            candidate.worktree.canonicalize().unwrap()
15759        );
15760        std::fs::remove_dir_all(&candidate.worktree).unwrap();
15761        assert_eq!(
15762            git::worktree_holding(&repo, &candidate.branch)
15763                .await
15764                .unwrap(),
15765            Some(registered)
15766        );
15767        assert!(
15768            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
15769            "the stale registration initially keeps the branch checked out"
15770        );
15771        runner.fold_losers().await.unwrap();
15772        assert!(runner.state.candidates[0].folded);
15773        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15774        assert_eq!(
15775            git::worktree_holding(&repo, &candidate.branch)
15776                .await
15777                .unwrap(),
15778            None
15779        );
15780        runner.state = RunState::load(&runner.state.id).unwrap();
15781        runner.fold_losers().await.unwrap();
15782    }
15783
15784    #[tokio::test]
15785    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
15786        let (_tmp, mut runner) = preservation_resume_fixture().await;
15787        let repo = runner.state.repo.clone();
15788        let candidate = runner.state.candidates[0].clone();
15789        let winner = runner.state.candidates[1].clone();
15790        let registered = git::worktree_holding(&repo, &candidate.branch)
15791            .await
15792            .unwrap()
15793            .unwrap();
15794        assert_eq!(
15795            registered.canonicalize().unwrap(),
15796            candidate.worktree.canonicalize().unwrap()
15797        );
15798        std::fs::remove_dir_all(&candidate.worktree).unwrap();
15799        assert_eq!(
15800            git::worktree_holding(&repo, &candidate.branch)
15801                .await
15802                .unwrap(),
15803            Some(registered)
15804        );
15805        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
15806
15807        for _ in 0..2 {
15808            fold_run(&mut runner.state, false, &crate::run::home())
15809                .await
15810                .unwrap();
15811            runner.state = RunState::load(&runner.state.id).unwrap();
15812            assert!(runner.state.candidates[0].folded);
15813            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15814            assert_eq!(
15815                git::worktree_holding(&repo, &candidate.branch)
15816                    .await
15817                    .unwrap(),
15818                None
15819            );
15820            assert!(!runner.state.candidates[1].folded);
15821            assert!(winner.worktree.exists());
15822            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
15823        }
15824    }
15825
15826    #[tokio::test]
15827    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
15828        ask_test_home();
15829        let tmp = tempfile::tempdir().unwrap();
15830        let repo = tmp.path().join("repo");
15831        std::fs::create_dir(&repo).unwrap();
15832        init_repo(&repo);
15833        let mut runner = runner_at(RunStatus::Implementing);
15834        runner.state.repo = repo.clone();
15835        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15836        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15837        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15838            let branch = format!("candidate-{label}");
15839            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
15840            git::worktree_add_branch(&repo, &wt, &branch, "main")
15841                .await
15842                .unwrap();
15843            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
15844            git::git(&wt, &["add", "README.md"]).await.unwrap();
15845            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
15846            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
15847            runner.state.candidates.push(Candidate {
15848                index,
15849                label,
15850                agent: "alpha".to_owned(),
15851                branch,
15852                worktree: wt,
15853                summary: "completed implementation".to_owned(),
15854                stat: String::new(),
15855                files: 0,
15856                commits: 0,
15857                empty: false,
15858                failed: None,
15859                verified_noop: None,
15860                duration_ms: 42,
15861                folded: false,
15862            });
15863            runner
15864                .state
15865                .candidate_preservation
15866                .insert(label, "awaiting persistence".to_owned());
15867        }
15868        let wt = runner.state.candidates[0].worktree.clone();
15869        let index_lock = git::git(
15870            &wt,
15871            &[
15872                "rev-parse",
15873                "--path-format=absolute",
15874                "--git-path",
15875                "index.lock",
15876            ],
15877        )
15878        .await
15879        .unwrap();
15880        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15881        runner.state.save().unwrap();
15882        assert!(runner.persist_candidates().await.is_err());
15883        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15884        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
15885        assert_eq!(
15886            runner.state.candidates[1].commits, 1,
15887            "other results are still collected"
15888        );
15889        assert!(
15890            !runner.state.candidates[0].empty,
15891            "failed rescue is not no change"
15892        );
15893        assert_eq!(
15894            runner.state.candidates[0].summary,
15895            "completed implementation"
15896        );
15897        assert!(runner.after_implement().is_err());
15898        let home = tmp.path().join("fold-home");
15899        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
15900        assert!(wt.join("new-file").exists());
15901        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
15902        assert!(recovery.join("staged.patch").exists());
15903        assert!(recovery.join("unstaged.patch").exists());
15904        assert_eq!(
15905            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
15906            "untracked edits\n"
15907        );
15908        runner.state.begin_pass();
15909        runner.state.end_pass_parked(ParkCause::Upgrade {
15910            from: "1.0".into(),
15911            to: Some("1.1".into()),
15912        });
15913        runner.state.save().unwrap();
15914        runner.state = RunState::load(&runner.state.id).unwrap();
15915        assert!(matches!(
15916            runner.state.passes.last().unwrap().end,
15917            Some(crate::run::PassEnd::Parked {
15918                cause: ParkCause::Upgrade { .. }
15919            })
15920        ));
15921        std::fs::remove_file(index_lock).unwrap();
15922        // No implementer is configured: a persistence retry cannot dispatch a seat.
15923        runner.implement().await.unwrap();
15924        assert!(runner.state.candidate_preservation.is_empty());
15925        assert!(
15926            runner
15927                .state
15928                .candidates
15929                .iter()
15930                .all(|c| c.commits == 1 && !c.empty)
15931        );
15932        assert!(git::is_clean(&wt).await.unwrap());
15933        assert!(
15934            crate::run::read_artifact(&runner.state, "cand-A.patch")
15935                .unwrap()
15936                .contains("later tracked edits")
15937        );
15938    }
15939
15940    #[tokio::test]
15941    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
15942        ask_test_home();
15943        let tmp = tempfile::tempdir().unwrap();
15944        let repo = tmp.path().join("repo");
15945        std::fs::create_dir(&repo).unwrap();
15946        init_repo(&repo);
15947        let wt = tmp.path().join("candidate");
15948        git::worktree_add_branch(&repo, &wt, "candidate", "main")
15949            .await
15950            .unwrap();
15951        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15952        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
15953        assert!(wt.join("uncommitted").exists());
15954        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15955        std::fs::remove_file(wt.join("uncommitted")).unwrap();
15956        let mut runner = runner_at(RunStatus::Implementing);
15957        runner.state.repo = repo.clone();
15958        runner.state.base_commit = "missing-base".to_owned();
15959        runner.state.candidates.push(Candidate {
15960            index: 0,
15961            label: 'A',
15962            agent: "alpha".into(),
15963            branch: "candidate".into(),
15964            worktree: wt.clone(),
15965            summary: "completed".into(),
15966            stat: String::new(),
15967            files: 0,
15968            commits: 0,
15969            empty: false,
15970            failed: None,
15971            verified_noop: None,
15972            duration_ms: 1,
15973            folded: false,
15974        });
15975        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15976        assert!(
15977            fold_run(&mut runner.state, true, &tmp.path().join("home"))
15978                .await
15979                .is_err()
15980        );
15981        assert!(!runner.state.candidates[0].folded);
15982        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15983        std::fs::remove_file(wt.join("uncommitted")).unwrap();
15984        runner
15985            .state
15986            .candidate_preservation
15987            .insert('A', "pending".into());
15988        assert!(runner.persist_candidates().await.is_err());
15989        assert!(!runner.state.candidates[0].empty);
15990        assert!(!runner.state.candidates[0].folded);
15991        assert!(wt.exists());
15992        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15993        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
15994        std::fs::create_dir_all(&patch_path).unwrap();
15995        assert!(
15996            runner.persist_candidates().await.is_err(),
15997            "artifact failure stays pending"
15998        );
15999        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16000        assert!(!runner.state.candidates[0].empty);
16001        std::fs::remove_dir(patch_path).unwrap();
16002        runner.persist_candidates().await.unwrap();
16003        assert!(
16004            runner.state.candidates[0].empty,
16005            "only successful inspection confirms no change"
16006        );
16007    }
16008
16009    fn init_repo(dir: &Path) {
16010        let run = |args: &[&str]| {
16011            let out = std::process::Command::new("git")
16012                .args(args)
16013                .current_dir(dir)
16014                .quiet()
16015                .output()
16016                .expect("spawn git");
16017            assert!(
16018                out.status.success(),
16019                "git {args:?} failed: {}",
16020                String::from_utf8_lossy(&out.stderr)
16021            );
16022        };
16023        run(&["init", "-b", "main"]);
16024        run(&["config", "user.name", "magi test"]);
16025        run(&["config", "user.email", "magi@example.com"]);
16026        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16027        run(&["add", "-A"]);
16028        run(&["commit", "-m", "init"]);
16029    }
16030
16031    // `settle_questions` is what closes the ghost the phone showed: a run's
16032    // seat asked something, the run then ended, and nothing was left to
16033    // abandon the question it left `open`. `HOME` is a process-wide
16034    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
16035    // first time it runs in the binary — every test below still reaches the
16036    // same directory whichever call won, and each gets its own run id from
16037    // `RunState::new`, so they never collide there.
16038    fn ask_test_home() {
16039        crate::run::pin_test_home();
16040    }
16041
16042    #[test]
16043    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16044        ask_test_home();
16045        let mut runner = runner_at(RunStatus::Prep);
16046        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16047        // A blocked temporary output path makes the terminal save fail while
16048        // preserving the already published reservation.
16049        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16050        let error = guard
16051            .finish(
16052                &mut runner.state,
16053                Err(
16054                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16055                        .into(),
16056                ),
16057            )
16058            .unwrap_err();
16059        assert_eq!(
16060            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16061            std::io::ErrorKind::PermissionDenied
16062        );
16063        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16064        let message = format!("{error:#}");
16065        assert!(message.contains("original refusal"));
16066        assert!(!message.contains("could not save terminal"));
16067        assert_eq!(
16068            error.chain().count(),
16069            2,
16070            "only startup attribution wraps the original error"
16071        );
16072        let saved = RunState::load(&runner.state.id).unwrap();
16073        assert_eq!(saved.status, RunStatus::Blocked);
16074        assert!(!saved.startup.unwrap().complete);
16075    }
16076
16077    #[test]
16078    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16079        ask_test_home();
16080        let mut runner = runner_at(RunStatus::Prep);
16081        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16082        let id = runner.state.id.clone();
16083        drop(guard);
16084        let saved = RunState::load(&id).unwrap();
16085        assert_eq!(saved.status, RunStatus::Blocked);
16086        assert!(!saved.startup.as_ref().unwrap().complete);
16087        assert!(
16088            saved
16089                .startup
16090                .unwrap()
16091                .failure
16092                .unwrap()
16093                .contains("interrupted")
16094        );
16095        assert!(Runner::resume(&id).is_err());
16096    }
16097
16098    #[test]
16099    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16100        ask_test_home();
16101        let mut runner = runner_at(RunStatus::Prep);
16102        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16103        let error = guard
16104            .finish(
16105                &mut runner.state,
16106                Err(
16107                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16108                        .into(),
16109                ),
16110            )
16111            .unwrap_err();
16112        assert_eq!(
16113            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16114            std::io::ErrorKind::PermissionDenied
16115        );
16116        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16117        let mut task = crate::queue::Task::new(
16118            "review".into(),
16119            "criteria".into(),
16120            runner.state.repo.clone(),
16121            crate::queue::Source::Human,
16122        );
16123        task.record_startup_failure(failure.clone());
16124        task.record_startup_failure(failure.clone());
16125        assert_eq!(task.startup_failures.len(), 1);
16126        assert!(task.runs.is_empty());
16127        assert_eq!(task.attempts, 0);
16128    }
16129
16130    #[cfg(unix)]
16131    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16132        ask_test_home();
16133        let tmp = tempfile::tempdir().unwrap();
16134        let repo = tmp.path().join("repo");
16135        std::fs::create_dir(&repo).unwrap();
16136        init_repo(&repo);
16137        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16138        let origin = tmp.path().join("origin.git");
16139        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16140            .await
16141            .unwrap();
16142        git::git(
16143            &repo,
16144            &["remote", "add", "origin", origin.to_str().unwrap()],
16145        )
16146        .await
16147        .unwrap();
16148        git::git(&repo, &["checkout", "-b", "fix/retained"])
16149            .await
16150            .unwrap();
16151        std::fs::write(repo.join("retained"), "retained change").unwrap();
16152        git::git(&repo, &["add", "."]).await.unwrap();
16153        git::git(&repo, &["commit", "-m", "retained implementation"])
16154            .await
16155            .unwrap();
16156        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16157        git::git(&repo, &["checkout", "main"]).await.unwrap();
16158        std::fs::write(repo.join("base-change"), "new base").unwrap();
16159        git::git(&repo, &["add", "."]).await.unwrap();
16160        git::git(&repo, &["commit", "-m", "base moved"])
16161            .await
16162            .unwrap();
16163        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16164        git::git(&repo, &["checkout", "fix/retained"])
16165            .await
16166            .unwrap();
16167        let mut runner = runner_at(RunStatus::Voting);
16168        runner.state.repo = repo.clone();
16169        runner.state.base_commit = base;
16170        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16171        runner.state.config.disk.min_free_bytes = 0;
16172        runner.state.config.graph.review_rounds = 1;
16173        runner.state.config.verify.e2e = vec!["true".into()];
16174        runner.state.config.verify.gate = vec!["true".into()];
16175        runner.state.candidates = vec![Candidate {
16176            index: 0,
16177            label: 'A',
16178            agent: "fixture".into(),
16179            branch: "fix/retained".into(),
16180            worktree: repo.clone(),
16181            summary: String::new(),
16182            stat: String::new(),
16183            files: 1,
16184            commits: 1,
16185            empty: false,
16186            failed: None,
16187            verified_noop: None,
16188            duration_ms: 0,
16189            folded: false,
16190        }];
16191        runner.state.tally = Some(Tally {
16192            first_choice: BTreeMap::new(),
16193            borda: BTreeMap::new(),
16194            winner: 'A',
16195            rankings: 0,
16196            unanimous_initial: true,
16197            deliberated: false,
16198            changed_votes: 0,
16199            unanimous_final: true,
16200            tie_break: None,
16201            judges: 0,
16202            present: 0,
16203            quorum: 0,
16204            met_quorum: true,
16205            uncontested: Some("review-only".into()),
16206        });
16207        let mut task = crate::queue::Task::new(
16208            "fixture".into(),
16209            "fixture".into(),
16210            repo,
16211            crate::queue::Source::Human,
16212        );
16213        task.link_run(&runner.state.id);
16214        crate::queue::Queue::open().put(&mut task).unwrap();
16215        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16216        runner.state.base_sync = Some(BaseSync {
16217            tip: String::new(),
16218            behind: 1,
16219            attempts: BASE_SYNC_ROUNDS,
16220            conflict: None,
16221            already_in: None,
16222        });
16223        runner.sync_to_base().await.unwrap();
16224        assert_eq!(runner.state.status, RunStatus::Blocked);
16225        (tmp, runner, head, task.id)
16226    }
16227
16228    #[cfg(unix)]
16229    #[tokio::test]
16230    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16231        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16232        if std::env::var_os(CHILD).is_none() {
16233            let home = tempfile::tempdir().unwrap();
16234            let output = std::process::Command::new(std::env::current_exe().unwrap())
16235                .args([
16236                    "--exact",
16237                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16238                    "--nocapture",
16239                ])
16240                .env(CHILD, "1")
16241                .env("MAGI_HOME", home.path())
16242                .env("MAGI_WORKER_HOME", home.path())
16243                .env("MAGI_WORKER_ID", "abcdef")
16244                .output()
16245                .unwrap();
16246            assert!(
16247                output.status.success()
16248                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16249                "{}\n{}",
16250                String::from_utf8_lossy(&output.stdout),
16251                String::from_utf8_lossy(&output.stderr)
16252            );
16253            return;
16254        }
16255        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16256        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16257        let home = crate::run::home();
16258        let queue = crate::queue::Queue::open();
16259        let mut owner = queue.get(&task).unwrap();
16260        owner.release_operator("test-release", None);
16261        owner.start(runner.state.id.clone());
16262        queue.put(&mut owner).unwrap();
16263        runner.state.driver_exited = true;
16264        runner.state.task_instruction = Some(owner.instruction.clone());
16265        runner.state.save().unwrap();
16266        let _claim = queue.claim(&task).unwrap();
16267        let reserve = |id: &str| {
16268            let directory = home.join("workers/jobs").join(id);
16269            std::fs::create_dir_all(&directory).unwrap();
16270            let job = crate::worker::Job {
16271                id: id.into(),
16272                sequence: 1,
16273                kind: "task".into(),
16274                key: format!("task:{task}"),
16275                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16276                generation: "fixture-generation".into(),
16277                version: None,
16278                status: crate::worker::Status::Running,
16279                result: None,
16280                pid: Some(std::process::id()),
16281                created_at: 0,
16282                finished_at: None,
16283            };
16284            std::fs::write(
16285                directory.join("job.json"),
16286                serde_json::to_vec(&job).unwrap(),
16287            )
16288            .unwrap();
16289            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16290        };
16291        let _own = reserve("abcdef");
16292        let other = reserve("fedcba");
16293        assert!(
16294            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16295                .await
16296                .is_err()
16297        );
16298        assert!(
16299            RunState::load(&runner.state.id)
16300                .unwrap()
16301                .base_sync_recovery
16302                .authorizations
16303                .is_empty()
16304        );
16305        drop(other);
16306        assert!(crate::daemon::is_working_on(
16307            &home,
16308            &runner.state.id,
16309            Timestamp::now()
16310        ));
16311        assert!(
16312            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16313        );
16314        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16315            .await
16316            .unwrap();
16317        assert!(
16318            RunState::load(&runner.state.id)
16319                .unwrap()
16320                .base_recovery_pending()
16321        );
16322    }
16323
16324    #[cfg(unix)]
16325    #[tokio::test]
16326    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16327        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16328        let home = crate::run::home();
16329        runner.state.driver_pid = Some(std::process::id());
16330        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16331        runner.state.driver_exited = false;
16332        runner.state.begin_pass();
16333        runner.state.save().unwrap();
16334        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16335        driver.record(&runner.state).unwrap();
16336        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16337        // Preserve the durable claim as if the process died before Drop saved.
16338        drop(driver);
16339        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16340        assert_eq!(
16341            runner.state.liveness_under(&home, false),
16342            crate::run::Liveness::Dead
16343        );
16344        let queue = crate::queue::Queue::open();
16345        let _claim = queue.claim(&task).unwrap();
16346        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16347            .await
16348            .unwrap();
16349        let disk = RunState::load(&runner.state.id).unwrap();
16350        assert!(!disk.driver_exited);
16351        assert!(disk.base_recovery_pending());
16352    }
16353
16354    #[cfg(unix)]
16355    #[tokio::test]
16356    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16357        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16358        if std::env::var_os(CHILD).is_none() {
16359            use std::os::unix::fs::PermissionsExt;
16360            let tmp = tempfile::tempdir().unwrap();
16361            let gh = tmp.path().join("gh");
16362            std::fs::write(&gh, r#"#!/bin/sh
16363case "$1 $2" in
16364  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16365  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16366  *) exit 2 ;;
16367esac
16368"#).unwrap();
16369            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16370            let mut paths = vec![tmp.path().to_path_buf()];
16371            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16372            let output = std::process::Command::new(std::env::current_exe().unwrap())
16373                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16374                .env(CHILD, "1")
16375                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16376                .env("PATH", std::env::join_paths(paths).unwrap())
16377                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16378                .output().unwrap();
16379            assert!(
16380                output.status.success()
16381                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16382                "{}\n{}",
16383                String::from_utf8_lossy(&output.stdout),
16384                String::from_utf8_lossy(&output.stderr)
16385            );
16386            return;
16387        }
16388        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16389        let repo = runner.state.repo.clone();
16390        git::git(&repo, &["checkout", "main"]).await.unwrap();
16391        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16392        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16393        git::git(&repo, &["checkout", "fix/retained"])
16394            .await
16395            .unwrap();
16396        runner.state.pr = Some(crate::run::PrRecord {
16397            url: "https://example.test/project/pull/1".into(),
16398            number: 1,
16399            state: "open".into(),
16400            checks: "unknown".into(),
16401            round: 0,
16402            rounds: 1,
16403            red_at_merge: Vec::new(),
16404        });
16405        runner.sync_to_base().await.unwrap();
16406        assert_eq!(runner.state.status, RunStatus::Blocked);
16407        assert_eq!(
16408            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16409            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16410        );
16411        assert!(runner.state.base_sync_closure_pending());
16412        assert!(crate::handover::can_resume(&runner.state));
16413        let queue = crate::queue::Queue::open();
16414        assert!(!crate::base_sync_recovery::held_task(
16415            &queue,
16416            &queue.get(&task).unwrap()
16417        ));
16418        assert!(runner.state.authorize_base_sync(&task).is_err());
16419        // Legacy closure stops have no typed stop but must retain the checkpoint.
16420        runner.state.base_sync_recovery.stop = None;
16421        runner.state.status = RunStatus::Voting;
16422        runner.state.gate_ran = true;
16423        runner.state.config.graph.review_rounds = 0;
16424        runner.state.save().unwrap();
16425        assert!(crate::handover::can_resume(&runner.state));
16426        runner.state = RunState::load(&runner.state.id).unwrap();
16427        runner.execute_graph().await.unwrap();
16428        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16429        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16430        assert_eq!(
16431            runner.state.base_sync.as_ref().unwrap().attempts,
16432            BASE_SYNC_ROUNDS
16433        );
16434        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16435        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16436        assert!(runner.state.reviews.is_empty());
16437        assert!(runner.state.gate.is_empty());
16438    }
16439
16440    #[cfg(unix)]
16441    #[tokio::test]
16442    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16443        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16444        runner.state.driver_exited = true;
16445        runner.state.save().unwrap();
16446        let home = crate::run::home();
16447        let queue = crate::queue::Queue::open();
16448        let _claim = queue.claim(&task).unwrap();
16449        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16450        assert!(
16451            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16452                .await
16453                .is_err()
16454        );
16455        drop(lease);
16456        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16457        assert!(
16458            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16459                .await
16460                .is_err()
16461        );
16462        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16463        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16464            .await
16465            .unwrap();
16466        let disk = RunState::load(&runner.state.id).unwrap();
16467        assert!(disk.base_recovery_pending());
16468        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16469        assert!(
16470            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16471                .await
16472                .is_err()
16473        );
16474        assert_eq!(
16475            RunState::load(&runner.state.id)
16476                .unwrap()
16477                .base_sync_recovery
16478                .authorizations
16479                .len(),
16480            1
16481        );
16482        let mut owner = queue.get(&task).unwrap();
16483        owner.runs.push("20260101-000000-ffff".into());
16484        queue.put(&mut owner).unwrap();
16485        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16486            .await
16487            .unwrap_err();
16488        assert!(error.to_string().contains("run task ownership changed"));
16489    }
16490
16491    #[cfg(unix)]
16492    #[tokio::test]
16493    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16494        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16495        let mut legacy = serde_json::to_value(&runner.state).unwrap();
16496        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16497        legacy["status"] = serde_json::json!("voting");
16498        runner.state = serde_json::from_value(legacy).unwrap();
16499        runner.state.save().unwrap();
16500        for _ in 0..3 {
16501            runner.execute_graph().await.unwrap();
16502            assert_eq!(runner.state.status, RunStatus::Blocked);
16503            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16504            assert!(!crate::handover::can_resume(&runner.state));
16505            assert_eq!(
16506                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16507                head
16508            );
16509            assert!(runner.state.reviews.is_empty());
16510            assert!(!runner.state.gate_ran);
16511        }
16512        let notice = crate::notices::run_ended(&runner.state).unwrap();
16513        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16514        runner.state.begin_pass();
16515        runner.state.status = RunStatus::Voting;
16516        runner.deliberate().await.unwrap();
16517        runner.vote().await.unwrap();
16518        runner.finish_after_tally().await.unwrap();
16519        assert_eq!(runner.state.status, RunStatus::Blocked);
16520        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16521        assert_eq!(
16522            crate::notices::run_ended(&runner.state).unwrap().since,
16523            notice.since
16524        );
16525    }
16526
16527    #[cfg(unix)]
16528    #[tokio::test]
16529    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16530        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16531        for detail in [
16532            "fix/retained conflicts with origin/main and did not rebase: conflict",
16533            "rebased fix/retained locally but origin refused the push",
16534            "unknown failure",
16535        ] {
16536            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16537            runner.state.record_base_sync_stop(&head);
16538            assert!(runner.state.authorize_base_sync(&task).is_err());
16539            runner.state.status = RunStatus::Voting;
16540            runner.execute_graph().await.unwrap();
16541            assert_eq!(runner.state.status, RunStatus::Blocked);
16542            assert!(!crate::handover::can_resume(&runner.state));
16543            assert_eq!(
16544                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16545                head
16546            );
16547        }
16548    }
16549
16550    #[cfg(unix)]
16551    #[tokio::test]
16552    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
16553        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16554        runner.state.authorize_base_sync(&task).unwrap();
16555        assert!(runner.state.consume_base_recovery(&head));
16556        runner.state.save().unwrap();
16557        runner.state = RunState::load(&runner.state.id).unwrap();
16558        runner.execute_graph().await.unwrap();
16559        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16560        assert_eq!(runner.state.status, RunStatus::Blocked);
16561        assert!(!runner.state.consume_base_recovery(&head));
16562        assert!(!crate::handover::can_resume(&runner.state));
16563        runner.state.authorize_base_sync(&task).unwrap();
16564        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
16565        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16566        assert_eq!(
16567            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16568            head
16569        );
16570    }
16571
16572    #[cfg(unix)]
16573    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
16574        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
16575        // This is an ordinary gate stop, not an authorized base-budget recovery.
16576        runner.state.prep_complete = Some(true);
16577        runner.state.base_sync = None;
16578        runner.state.base_sync_recovery = Default::default();
16579        runner.state.config.graph.gate_fix_rounds = 1;
16580        let mut review = review_round(true, 0, 1, 1, false, true);
16581        review.head = head.clone();
16582        review.verified_head = Some(head.clone());
16583        runner.state.reviews = vec![review];
16584        runner.state.gate = vec![CommandOutcome {
16585            command: "old gate".into(),
16586            code: Some(1),
16587            output_tail: "old failure".into(),
16588            duration_ms: 0,
16589            resource_blocked: false,
16590        }];
16591        runner.state.gate_head = Some(head.clone());
16592        runner.state.gate_ran = true;
16593        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
16594            agent: "fixture".into(),
16595            failed: runner.state.gate.clone(),
16596            notes: "spent".into(),
16597            committed: false,
16598            error: None,
16599        }];
16600        runner.state.land_approval = Some(crate::run::LandApproval {
16601            question: "old approval".into(),
16602            head: head.clone(),
16603        });
16604        runner.state.status = RunStatus::Blocked;
16605        (tmp, runner, head)
16606    }
16607
16608    #[cfg(unix)]
16609    #[tokio::test]
16610    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
16611        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
16612            let (tmp, mut runner, old) = resumed_gate_fixture().await;
16613            let phases = tmp.path().join("phases");
16614            runner.state.config.verify.e2e = vec![format!(
16615                "echo e2e >> '{}'; {}",
16616                phases.display(),
16617                if e2e_ok {
16618                    "true"
16619                } else {
16620                    "echo new-e2e-failure; false"
16621                }
16622            )];
16623            runner.state.config.verify.gate = vec![format!(
16624                "echo gate >> '{}'; {}",
16625                phases.display(),
16626                if gate_ok {
16627                    "true"
16628                } else {
16629                    "echo new-gate-failure; false"
16630                }
16631            )];
16632            runner.state.save().unwrap();
16633            runner.state = RunState::load(&runner.state.id).unwrap();
16634            runner.sync_to_base().await.unwrap();
16635            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16636            assert_ne!(head, old);
16637            // Reentry after Git updated but before verification was invalidated.
16638            runner.state = RunState::load(&runner.state.id).unwrap();
16639            runner.execute_graph().await.unwrap();
16640            assert_eq!(runner.state.gate_fixes.len(), 1);
16641            assert_eq!(runner.state.reviews.len(), 1);
16642            assert_eq!(runner.state.reviews[0].head, old);
16643            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
16644            assert_eq!(runner.state.verification_history.len(), 1);
16645            let history = &runner.state.verification_history[0];
16646            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
16647            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
16648            assert!(history.archived_at.is_some());
16649            assert_eq!(history.fixes_spent, Some(1));
16650            assert_eq!(history.gate[0].output_tail, "old failure");
16651            let archive_events: Vec<_> = runner
16652                .state
16653                .events
16654                .iter()
16655                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
16656                .collect();
16657            assert_eq!(archive_events.len(), 1);
16658            assert!(archive_events[0].message.contains(&short(&old)));
16659            assert!(archive_events[0].message.contains(&short(&head)));
16660            assert!(
16661                archive_events[0]
16662                    .message
16663                    .contains("1 gate-fix round(s) already spent stay spent")
16664            );
16665            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
16666            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
16667            if e2e_ok {
16668                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16669                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
16670            } else {
16671                assert!(!runner.state.gate_ran);
16672            }
16673            assert_eq!(
16674                runner.state.status,
16675                if e2e_ok && gate_ok {
16676                    RunStatus::Ready
16677                } else {
16678                    RunStatus::Blocked
16679                }
16680            );
16681            if !e2e_ok || !gate_ok {
16682                let cause = runner.state.verification_blocking_cause().unwrap();
16683                assert!(cause.contains(&head));
16684                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
16685                // A later successful progress event must not become the cause.
16686                runner.state.event("land", "rebased successfully");
16687                let notice = crate::notices::run_ended(&runner.state).unwrap();
16688                assert!(!format!("{notice:?}").contains("rebased successfully"));
16689                assert!(runner.state.merge.is_none());
16690            }
16691            let before = std::fs::read_to_string(&phases).unwrap();
16692            runner.state.save().unwrap();
16693            runner.state = RunState::load(&runner.state.id).unwrap();
16694            runner.review_loop().await.unwrap();
16695            runner.gate().await.unwrap();
16696            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
16697            assert_eq!(runner.state.verification_history.len(), 1);
16698            let history = &runner.state.verification_history[0];
16699            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
16700            assert!(history.archived_at.is_some());
16701            assert_eq!(history.fixes_spent, Some(1));
16702        }
16703    }
16704
16705    #[cfg(unix)]
16706    #[tokio::test]
16707    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
16708        let (tmp, mut runner, head) = resumed_gate_fixture().await;
16709        let marker = tmp.path().join("gate-ran");
16710        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
16711        runner.review_loop().await.unwrap();
16712        runner.gate().await.unwrap();
16713        assert_eq!(runner.state.status, RunStatus::Blocked);
16714        assert!(!marker.exists());
16715        assert!(runner.state.verification_history.is_empty());
16716        runner.state.gate_head = None;
16717        runner.gate().await.unwrap();
16718        assert!(marker.exists());
16719        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16720        assert_eq!(runner.state.verification_history.len(), 1);
16721        assert_eq!(runner.state.gate_fixes.len(), 1);
16722    }
16723
16724    #[cfg(unix)]
16725    #[tokio::test]
16726    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
16727        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16728        runner.sync_to_base().await.unwrap();
16729        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16730        runner.state.reconcile_verification_head(&head);
16731        // Both crash-before-command and resource wait are unresolved evidence.
16732        for outcomes in [
16733            Vec::new(),
16734            vec![CommandOutcome {
16735                command: "true".into(),
16736                code: None,
16737                output_tail: "cache unavailable".into(),
16738                duration_ms: 0,
16739                resource_blocked: true,
16740            }],
16741        ] {
16742            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16743                head: head.clone(),
16744                passed: false,
16745                outcomes,
16746            });
16747            runner.state.gate_ran = false;
16748            runner.state.gate.clear();
16749            runner.state.save().unwrap();
16750            runner.state = RunState::load(&runner.state.id).unwrap();
16751            runner.merge().await.unwrap();
16752            assert!(runner.state.merge.is_none());
16753            runner.review_loop().await.unwrap();
16754            runner.gate().await.unwrap();
16755            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
16756            assert!(runner.state.gate_allows_progress());
16757            assert_eq!(runner.state.gate_fixes.len(), 1);
16758            assert!(runner.state.merge.is_none());
16759        }
16760        // Moving the winner after a successful gate cannot publish an unchecked SHA.
16761        git::git(
16762            &runner.state.repo,
16763            &["commit", "--allow-empty", "-m", "external update"],
16764        )
16765        .await
16766        .unwrap();
16767        runner.merge().await.unwrap();
16768        assert!(runner.state.merge.is_none());
16769        assert_eq!(runner.state.status, RunStatus::Blocked);
16770    }
16771
16772    #[cfg(unix)]
16773    #[tokio::test]
16774    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
16775        for waiting_on_gate in [false, true] {
16776            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
16777            runner.sync_to_base().await.unwrap();
16778            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16779            runner.state.reconcile_verification_head(&head);
16780            if waiting_on_gate {
16781                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16782                    head: head.clone(),
16783                    passed: true,
16784                    outcomes: Vec::new(),
16785                });
16786            }
16787            let cache = tmp.path().join("cache");
16788            runner.state.config.verify.e2e =
16789                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
16790            runner.state.config.verify.gate = vec!["true".into()];
16791            runner.state.config.graph.timeout_resource_wait = 1;
16792            let owner =
16793                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
16794            let guard =
16795                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
16796                    crate::cache::AcquireOutcome::Acquired(g) => g,
16797                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
16798                };
16799            runner.review_loop().await.unwrap();
16800            runner.gate().await.unwrap();
16801            assert!(!runner.state.gate_ran);
16802            assert_eq!(runner.state.gate_fixes.len(), 1);
16803            assert!(runner.state.resource_block.is_some());
16804            runner.merge().await.unwrap();
16805            assert!(runner.state.merge.is_none());
16806            drop(guard);
16807            runner.state = RunState::load(&runner.state.id).unwrap();
16808            // The driver clears the previous pass's resource stop on resume.
16809            runner.state.resource_block = None;
16810            runner.review_loop().await.unwrap();
16811            runner.gate().await.unwrap();
16812            assert!(runner.state.gate_allows_progress());
16813            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16814            assert_eq!(runner.state.gate_fixes.len(), 1);
16815        }
16816    }
16817
16818    #[cfg(unix)]
16819    #[tokio::test]
16820    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
16821        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16822        runner.sync_to_base().await.unwrap();
16823        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16824        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
16825        // Verification passes before pre_gate and fails on the mechanically changed head.
16826        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
16827        runner.review_loop().await.unwrap();
16828        runner.gate().await.unwrap();
16829        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16830        assert_ne!(before, after);
16831        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
16832        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
16833        assert!(!runner.state.gate_allows_progress());
16834        runner.merge().await.unwrap();
16835        assert!(runner.state.merge.is_none());
16836        assert!(
16837            runner
16838                .state
16839                .verification_history
16840                .iter()
16841                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
16842        );
16843
16844        // An incomplete review is still an independent hold on a changed head.
16845        runner.state.reviews[0].clean = false;
16846        runner.state.reviews[0].answered = 0;
16847        runner.state.config.verify.e2e = vec!["true".into()];
16848        runner.review_loop().await.unwrap();
16849        runner.gate().await.unwrap();
16850        runner.merge().await.unwrap();
16851        assert_eq!(runner.state.status, RunStatus::Blocked);
16852        assert!(runner.state.merge.is_none());
16853        assert!(
16854            runner
16855                .state
16856                .verification_blocking_cause()
16857                .unwrap()
16858                .contains("review blocked")
16859        );
16860    }
16861
16862    #[cfg(unix)]
16863    #[tokio::test]
16864    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
16865        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
16866        runner.state.gate_head = None;
16867        runner.state.config.verify.gate =
16868            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
16869        runner.review_loop().await.unwrap();
16870        runner.gate().await.unwrap();
16871        assert_ne!(
16872            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16873            old
16874        );
16875        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
16876        assert!(!runner.state.gate_allows_progress());
16877        assert_eq!(runner.state.status, RunStatus::Blocked);
16878        runner.merge().await.unwrap();
16879        assert!(runner.state.merge.is_none());
16880    }
16881
16882    #[cfg(unix)]
16883    #[tokio::test]
16884    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
16885        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16886        let mut old_review = review_round(true, 0, 1, 1, false, true);
16887        old_review.head = head.clone();
16888        old_review.verified_head = Some(head.clone());
16889        runner.state.reviews = vec![old_review];
16890        runner.state.gate = vec![CommandOutcome {
16891            command: "old gate".into(),
16892            code: None,
16893            output_tail: "timed out after 2400s".into(),
16894            duration_ms: 2_400_000,
16895            resource_blocked: false,
16896        }];
16897        runner.state.gate_ran = true;
16898        runner.state.gate_head = Some(head.clone());
16899        runner.state.land_approval = Some(crate::run::LandApproval {
16900            question: "old-head-approval".into(),
16901            head: head.clone(),
16902        });
16903        runner.state.authorize_base_sync(&task).unwrap();
16904        runner.sync_to_base().await.unwrap();
16905        assert!(!runner.state.base_sync_blocked());
16906        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16907        assert_ne!(head, fresh);
16908        assert!(runner.state.reviews.is_empty());
16909        assert!(!runner.state.gate_ran);
16910        let saved = &runner.state.base_sync_recovery.authorizations[0];
16911        assert_eq!(saved.reviews[0].head, head);
16912        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
16913        assert!(saved.gate_ran);
16914        assert!(!saved.gate[0].ok());
16915        runner.review_loop().await.unwrap();
16916        runner.gate().await.unwrap();
16917        assert_eq!(runner.state.reviews[0].head, fresh);
16918        assert_eq!(
16919            runner.state.reviews[0].verified_head.as_deref(),
16920            Some(fresh.as_str())
16921        );
16922        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
16923        assert!(runner.state.gate_ran);
16924        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16925        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
16926        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
16927        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16928        assert_eq!(
16929            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
16930            "retained change"
16931        );
16932    }
16933
16934    #[cfg(unix)]
16935    #[tokio::test]
16936    async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
16937        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16938        let queue = crate::queue::Queue::open();
16939        let mut owner = queue.get(&task).unwrap();
16940        owner.release_operator("test-resume", None);
16941        queue.put(&mut owner).unwrap();
16942        let generation = owner.resumption.as_ref().unwrap().generation;
16943        runner.state.task_instruction = Some(owner.instruction.clone());
16944        runner.state.driver_exited = true;
16945        runner.state.save().unwrap();
16946        let home = crate::run::home();
16947        let _claim = queue.claim(&task).unwrap();
16948        owner.resumption.as_mut().unwrap().at =
16949            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
16950        queue.put(&mut owner).unwrap();
16951        assert!(
16952            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16953                .await
16954                .is_err(),
16955            "an approval from before the stop grants nothing"
16956        );
16957        owner.resumption.as_mut().unwrap().at = Timestamp::now();
16958        queue.put(&mut owner).unwrap();
16959        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16960            .await
16961            .unwrap();
16962        assert!(
16963            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16964                .await
16965                .is_err()
16966        );
16967        runner.state = RunState::load(&runner.state.id).unwrap();
16968        assert_eq!(
16969            runner.state.base_sync_recovery.authorizations[0].generation,
16970            Some(generation)
16971        );
16972        runner.sync_to_base().await.unwrap();
16973        let repo = runner.state.repo.clone();
16974        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16975        git::git(&repo, &["checkout", "main"]).await.unwrap();
16976        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
16977        git::git(&repo, &["add", "."]).await.unwrap();
16978        git::git(&repo, &["commit", "-m", "move base again"])
16979            .await
16980            .unwrap();
16981        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16982        git::git(&repo, &["checkout", "fix/retained"])
16983            .await
16984            .unwrap();
16985        runner.finish_after_tally().await.unwrap();
16986        assert_eq!(runner.state.status, RunStatus::Blocked);
16987        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16988        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16989        assert!(!runner.state.base_recovery_pending());
16990        assert!(!crate::handover::can_resume(&runner.state));
16991        assert!(!runner.state.gate_ran);
16992        // Even replaying the generation with a later timestamp grants nothing.
16993        owner.resumption.as_mut().unwrap().at = Timestamp::now();
16994        queue.put(&mut owner).unwrap();
16995        assert!(
16996            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16997                .await
16998                .is_err()
16999        );
17000        runner.state = RunState::load(&runner.state.id).unwrap();
17001        runner.sync_to_base().await.unwrap();
17002        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17003        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
17004        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17005    }
17006
17007    #[cfg(unix)]
17008    #[tokio::test]
17009    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
17010        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17011        runner.state.base_sync = None;
17012        runner.state.status = RunStatus::Blocked;
17013        let mut second = runner.state.candidates[0].clone();
17014        second.label = 'B';
17015        runner.state.candidates.push(second);
17016        assert!(runner.state.votes.is_empty());
17017        assert!(runner.state.deliberation.is_empty());
17018        runner.deliberate().await.unwrap();
17019        runner.vote().await.unwrap();
17020        assert_eq!(runner.state.status, RunStatus::Blocked);
17021        assert!(runner.state.votes.is_empty());
17022        assert!(runner.state.deliberation.is_empty());
17023    }
17024
17025    /// A minimal, git-free `Runner` at a given status — `settle_questions`
17026    /// reads nothing else off it.
17027    fn runner_at(status: RunStatus) -> Runner {
17028        let mut state = RunState::new(
17029            PathBuf::from("/nonexistent/repo"),
17030            "main".to_owned(),
17031            "deadbeef".to_owned(),
17032            "task".to_owned(),
17033            Config::default(),
17034        );
17035        // These fixtures share one home and save concurrently. Short clock
17036        // IDs can collide, making an ownership test read another test's run.
17037        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
17038        state.id = format!(
17039            "20000101-000000-{:04x}",
17040            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
17041        );
17042        state.status = status;
17043        Runner {
17044            state,
17045            roles: ResolvedRoles {
17046                implementers: Vec::new(),
17047                judges: Vec::new(),
17048                reviewers: Vec::new(),
17049                fixer: None,
17050                conductor: conductor(),
17051                implementer_roster: Vec::new(),
17052                judge_roster: Vec::new(),
17053                reviewer_roster: Vec::new(),
17054            },
17055            sem: Arc::new(Semaphore::new(1)),
17056            pause: Pause::new(),
17057            interrupt: Pause::new(),
17058        }
17059    }
17060
17061    #[tokio::test]
17062    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
17063        crate::run::pin_test_home();
17064        let mut runner = runner_at(RunStatus::Gating);
17065        runner.state.save().unwrap();
17066        let id = runner.state.id.clone();
17067        let passes = runner.state.passes.len();
17068        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
17069        let mut claiming = Box::pin(runner.claim_driver());
17070        std::future::poll_fn(|cx| {
17071            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
17072            std::task::Poll::Ready(())
17073        })
17074        .await;
17075        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
17076        drop(owner);
17077        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
17078            .await
17079            .unwrap()
17080            .unwrap();
17081        assert_eq!(execution.runner.state.passes.len(), passes + 1);
17082        assert!(execution.runner.state.resource_block.is_none());
17083        assert_eq!(
17084            execution.runner.state.liveness(false),
17085            crate::run::Liveness::Live
17086        );
17087        drop(execution);
17088        assert!(RunState::load(&id).unwrap().driver_exited);
17089    }
17090
17091    #[tokio::test]
17092    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
17093        crate::run::pin_test_home();
17094        let mut runner = runner_at(RunStatus::Gating);
17095        runner.state.save().unwrap();
17096        // run.json can commit, but the subsequent lease metadata write fails.
17097        // Cleanup must run after releasing the short ownership lock.
17098        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
17099        assert!(runner.claim_driver().await.is_err());
17100        assert!(runner.state.driver_exited);
17101        assert_eq!(runner.state.driver_pid, None);
17102        let disk = RunState::load(&runner.state.id).unwrap();
17103        assert!(disk.driver_exited);
17104        assert_eq!(disk.driver_pid, None);
17105        assert_eq!(disk.status, RunStatus::Gating);
17106    }
17107
17108    #[tokio::test]
17109    async fn an_early_graph_error_releases_its_driver() {
17110        crate::run::pin_test_home();
17111        let mut runner = runner_at(RunStatus::Blocked);
17112        runner.state.config.graph.selected_implementer = Some("selected".into());
17113        runner.state.save().unwrap();
17114        assert!(runner.execute().await.is_err());
17115        let disk = RunState::load(&runner.state.id).unwrap();
17116        assert!(disk.driver_exited);
17117        assert_eq!(disk.driver_pid, None);
17118        assert_eq!(disk.status, RunStatus::Blocked);
17119    }
17120
17121    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
17122        crate::run::MergeRepair {
17123            question: "repair-question".into(),
17124            head: "old-head".into(),
17125            pr: "https://github.com/o/r/pull/42".into(),
17126            reviews: vec![review_round(false, 1, 1, 1, false, true)],
17127            gate: vec![outcome(Some(0), false)],
17128            gate_head: Some("old-head".into()),
17129            gate_fixes: Vec::new(),
17130            pre_gate: Vec::new(),
17131            phase,
17132            failure: None,
17133        }
17134    }
17135
17136    #[tokio::test]
17137    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
17138        crate::run::pin_test_home();
17139        let mut runner = runner_at(RunStatus::Landing);
17140        runner.state.merge_repairs.push(merge_repair_request(
17141            crate::run::MergeRepairPhase::Requested,
17142        ));
17143        runner.state.save().unwrap();
17144        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
17145        assert!(
17146            runner
17147                .execute()
17148                .await
17149                .unwrap_err()
17150                .to_string()
17151                .contains("already running")
17152        );
17153        assert_eq!(
17154            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
17155            crate::run::MergeRepairPhase::Requested
17156        );
17157        drop(claim);
17158        let mut disk = runner.state.clone();
17159        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
17160        disk.save().unwrap();
17161        assert!(
17162            runner
17163                .execute()
17164                .await
17165                .unwrap_err()
17166                .to_string()
17167                .contains("decision changed")
17168        );
17169        assert_eq!(
17170            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
17171            crate::run::MergeRepairPhase::Complete
17172        );
17173    }
17174
17175    #[test]
17176    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
17177        crate::run::pin_test_home();
17178        let mut runner = runner_at(RunStatus::Landing);
17179        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
17180        runner.state.reviews = request.reviews.clone();
17181        runner.state.gate = request.gate.clone();
17182        runner.state.gate_ran = true;
17183        runner.state.land_approval = Some(crate::run::LandApproval {
17184            question: "old".into(),
17185            head: "old-head".into(),
17186        });
17187        runner.state.merge_repairs.push(request);
17188        runner.begin_merge_repair_review(0).unwrap();
17189        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
17190        assert!(!runner.state.gate_status().ok());
17191        assert!(runner.state.land_approval.is_none());
17192        let disk = RunState::load(&runner.state.id).unwrap();
17193        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
17194        assert!(disk.merge_repairs[0].gate[0].ok());
17195        assert_eq!(
17196            disk.merge_repairs[0].phase,
17197            crate::run::MergeRepairPhase::Reviewing
17198        );
17199        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
17200        assert!(disk.reviews.is_empty());
17201    }
17202
17203    #[tokio::test]
17204    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
17205        crate::run::pin_test_home();
17206        for phase in [
17207            crate::run::MergeRepairPhase::Fixing,
17208            crate::run::MergeRepairPhase::Requested,
17209        ] {
17210            let mut runner = runner_at(RunStatus::Landing);
17211            runner.state.config.graph.land_rounds = 0;
17212            runner.state.merge_repairs.push(merge_repair_request(phase));
17213            runner.state.save().unwrap();
17214            assert!(!runner.repair_merge_review().await.unwrap());
17215            assert_eq!(runner.state.status, RunStatus::Blocked);
17216            assert_eq!(
17217                runner.state.merge_repairs[0].phase,
17218                crate::run::MergeRepairPhase::Failed
17219            );
17220            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
17221            assert!(
17222                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
17223                    "interrupted"
17224                } else {
17225                    "budget exhausted"
17226                })
17227            );
17228            assert!(runner.state.merge.is_none());
17229        }
17230    }
17231
17232    /// `park_here` folding in the reason `Pause::park_because` recorded -
17233    /// this is what lets an operator reading a run's events tell an
17234    /// interrupt-driven park from an ordinary shutdown park.
17235    #[test]
17236    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
17237        crate::run::pin_test_home();
17238        let mut runner = runner_at(RunStatus::Implementing);
17239        let interrupt = Pause::new();
17240        runner.watch_interrupt(interrupt.clone());
17241
17242        interrupt.park_because("task a1b2 asked to run first");
17243
17244        assert!(runner.park_here().expect("park_here"));
17245        assert!(runner.state.parked);
17246        let last = runner.state.events.last().expect("a park event");
17247        assert_eq!(last.node, "park");
17248        assert!(
17249            last.message.contains("task a1b2 asked to run first"),
17250            "expected the interrupt reason in {:?}",
17251            last.message
17252        );
17253    }
17254
17255    /// The cause is stored on the pass when it parks, and the first cause a
17256    /// shared pause was given is the one that stays.
17257    #[test]
17258    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
17259        crate::run::pin_test_home();
17260        let mut runner = runner_at(RunStatus::Implementing);
17261        runner.state.begin_pass();
17262        let shutdown = Pause::new();
17263        runner.on_pause(shutdown.clone());
17264        let upgrade = ParkCause::Upgrade {
17265            from: "0.121.0".to_owned(),
17266            to: Some("0.121.1".to_owned()),
17267        };
17268        shutdown.park_for(upgrade.clone());
17269        shutdown.park_for(ParkCause::Operator);
17270
17271        assert!(runner.park_here().expect("park_here"));
17272        assert_eq!(
17273            runner.state.passes[0].end,
17274            Some(crate::run::PassEnd::Parked { cause: upgrade })
17275        );
17276        let last = runner.state.events.last().expect("a park event");
17277        assert!(last.message.contains("0.121.0"), "{}", last.message);
17278    }
17279
17280    #[tokio::test]
17281    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
17282        crate::run::pin_test_home();
17283        let mut runner = runner_at(RunStatus::Reviewing);
17284        runner.state.save().unwrap();
17285        let mut execution = runner.claim_driver().await.expect("first execution");
17286        assert_eq!(execution.runner.state.passes.len(), 1);
17287        execution
17288            .driver
17289            .finish(&mut execution.runner.state)
17290            .await
17291            .unwrap();
17292    }
17293
17294    #[tokio::test]
17295    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
17296        crate::run::pin_test_home();
17297        let mut runner = runner_at(RunStatus::Reviewing);
17298        runner.state.config.agents = vec![conductor()];
17299        runner.state.config.graph.timeout_resource_wait = 0;
17300        runner.state.save().unwrap();
17301        let id = runner.state.id.clone();
17302        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
17303        assert!(runner.claim_driver().await.is_err());
17304        drop(owner);
17305        drop(runner);
17306        let mut resumed = Runner::resume(&id).unwrap();
17307        assert_eq!(resumed.state.driver_claimed, Some(false));
17308        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
17309        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
17310        let mut execution = resumed
17311            .claim_driver()
17312            .await
17313            .expect("retry initial claim after reload");
17314        assert_eq!(execution.runner.state.driver_claimed, Some(true));
17315        assert_eq!(execution.runner.state.passes.len(), 1);
17316        execution
17317            .driver
17318            .finish(&mut execution.runner.state)
17319            .await
17320            .unwrap();
17321    }
17322
17323    #[tokio::test]
17324    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
17325        crate::run::pin_test_home();
17326        let mut runner = runner_at(RunStatus::Reviewing);
17327        runner.state.config.agents = vec![conductor()];
17328        runner.state.save().unwrap();
17329        let id = runner.state.id.clone();
17330        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
17331        let mut resumed = Runner::resume(&id).unwrap();
17332        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
17333        assert!(resumed.claim_driver().await.is_err());
17334        assert!(RunState::load(&id).unwrap().passes.is_empty());
17335        drop(lease);
17336        let mut execution = resumed.claim_driver().await.unwrap();
17337        execution
17338            .driver
17339            .finish(&mut execution.runner.state)
17340            .await
17341            .unwrap();
17342    }
17343
17344    #[tokio::test]
17345    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
17346        crate::run::pin_test_home();
17347        let mut runner = runner_at(RunStatus::Reviewing);
17348        let state = &mut runner.state;
17349        state.config.agents = vec![conductor()];
17350        state.driver_claimed = None;
17351        state.driver_pid = Some(std::process::id());
17352        state.driver_started_at = None;
17353        state.begin_pass();
17354        state.save().unwrap();
17355        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
17356        std::fs::write(
17357            state.dir().join("driver.json"),
17358            serde_json::to_vec(&serde_json::json!({
17359                "pid": state.driver_pid, "started_at": null,
17360                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
17361            }))
17362            .unwrap(),
17363        )
17364        .unwrap();
17365        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
17366        drop(lease);
17367        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
17368        let mut resumed = Runner::resume(&state.id).unwrap();
17369        assert!(!resumed.state.driver_exited);
17370        let mut execution = resumed
17371            .claim_driver()
17372            .await
17373            .expect("lease proves previous execution ended");
17374        assert_eq!(execution.runner.state.passes.len(), 2);
17375        execution
17376            .driver
17377            .finish(&mut execution.runner.state)
17378            .await
17379            .unwrap();
17380    }
17381
17382    #[tokio::test]
17383    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
17384        crate::run::pin_test_home();
17385        let mut runner = runner_at(RunStatus::Reviewing);
17386        let mut json = serde_json::to_value(&runner.state).unwrap();
17387        json.as_object_mut().unwrap().remove("driver_claimed");
17388        runner.state = serde_json::from_value(json).unwrap();
17389        assert_eq!(runner.state.driver_claimed, None);
17390        runner.state.save().unwrap();
17391        let mut execution = runner
17392            .claim_driver()
17393            .await
17394            .expect("the execution lease, not a missing marker, excludes a second driver");
17395        assert_eq!(execution.runner.state.passes.len(), 1);
17396        execution
17397            .driver
17398            .finish(&mut execution.runner.state)
17399            .await
17400            .unwrap();
17401    }
17402
17403    #[tokio::test]
17404    async fn driver_admission_refuses_a_proven_live_driver() {
17405        crate::run::pin_test_home();
17406        let mut runner = runner_at(RunStatus::Reviewing);
17407        let pid = std::process::id();
17408        runner.state.driver_pid = Some(pid);
17409        runner.state.driver_started_at = crate::proc::process_started_at(pid);
17410        assert!(runner.state.driver_started_at.is_some());
17411        runner.state.save().unwrap();
17412        let error = runner
17413            .claim_driver()
17414            .await
17415            .err()
17416            .expect("a live driver must refuse");
17417        assert!(error.to_string().contains("live driver"), "{error:#}");
17418        let saved = RunState::load(&runner.state.id).unwrap();
17419        assert_eq!(saved.driver_pid, Some(pid));
17420        assert_eq!(saved.status, RunStatus::Reviewing);
17421        assert!(saved.passes.is_empty());
17422    }
17423
17424    #[test]
17425    fn parking_resource_halt_persists_the_pass_once() {
17426        crate::run::pin_test_home();
17427        let mut runner = runner_at(RunStatus::Reviewing);
17428        runner.state.begin_pass();
17429        runner.state.resource_block = Some(crate::run::ResourceBlock {
17430            node: "fix".into(),
17431            kind: "shared build cache".into(),
17432            detail: "cancelled".into(),
17433            at: Timestamp::now(),
17434            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
17435        });
17436        let cause = ParkCause::Upgrade {
17437            from: "old".into(),
17438            to: Some("new".into()),
17439        };
17440        runner.pause.park_for(cause.clone());
17441        // Both direct fixer exits and graph resource boundaries must complete
17442        // parking, rather than returning a stale Reviewing record.
17443        runner.stop_for_resource().unwrap();
17444        assert!(runner.resource_halt().unwrap());
17445        assert!(runner.park_here().unwrap());
17446        let saved = RunState::load(&runner.state.id).unwrap();
17447        assert!(saved.parked);
17448        assert_eq!(saved.status, RunStatus::Reviewing);
17449        assert!(saved.passes[0].ended_at.is_some());
17450        assert_eq!(
17451            saved.passes[0].end,
17452            Some(crate::run::PassEnd::Parked {
17453                cause: cause.clone()
17454            })
17455        );
17456        assert_eq!(
17457            saved.resource_block.unwrap().cause,
17458            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
17459        );
17460        assert_eq!(
17461            saved
17462                .events
17463                .iter()
17464                .filter(|event| event.node == "park")
17465                .count(),
17466            1
17467        );
17468    }
17469
17470    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
17471    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
17472    /// cleared) must not make a *different* run - one only watching its own,
17473    /// unshared interrupt `Pause` - see itself as parked. If a future change
17474    /// ever collapsed these back into one handle, the interrupt scheduler
17475    /// would park every run for the rest of the daemon's life, not just the
17476    /// one it meant to interrupt.
17477    #[test]
17478    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
17479        crate::run::pin_test_home();
17480        let mut runner = runner_at(RunStatus::Implementing);
17481        let shutdown = Pause::new();
17482        runner.on_pause(shutdown.clone());
17483        let interrupt = Pause::new();
17484        runner.watch_interrupt(interrupt.clone());
17485
17486        // Nobody has asked for anything yet.
17487        assert!(!runner.park_here().expect("park_here"));
17488        assert!(!runner.state.parked);
17489
17490        // Only the interrupt handle fires; the shutdown handle stays clear.
17491        interrupt.park_because("test");
17492        assert!(!shutdown.parked());
17493        assert!(runner.park_here().expect("park_here"));
17494    }
17495
17496    /// The property every prior attempt at this feature failed to pin down:
17497    /// asking a run to park while one of its nodes has a real, in-flight
17498    /// async operation running (an agent call, in production) must not cut
17499    /// that operation short. `park_here` is only ever consulted *between*
17500    /// `execute`'s node calls - see its own doc - so nothing inside a node
17501    /// can observe a park request until the node itself returns. This proves
17502    /// that structurally, with real `tokio` concurrency and a channel
17503    /// handshake (never a sleep, which would only prove "usually", not
17504    /// "cannot"): the "node" below reports that it has genuinely started,
17505    /// and only then is the park requested; the node still has to be told to
17506    /// finish before `park_here` is ever called, exactly mirroring every
17507    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
17508    /// in `execute`.
17509    #[tokio::test]
17510    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
17511        crate::run::pin_test_home();
17512        let mut runner = runner_at(RunStatus::Implementing);
17513        let interrupt = Pause::new();
17514        runner.watch_interrupt(interrupt.clone());
17515
17516        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
17517        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
17518
17519        // Stands in for one node's in-flight agent call: it proves it has
17520        // genuinely started, then blocks - exactly as a spawned CLI process
17521        // does - until told to finish.
17522        let node = async move {
17523            started_tx.send(()).expect("send started");
17524            finish_rx.await.expect("recv finish");
17525            "node finished"
17526        };
17527
17528        let interrupter = async move {
17529            started_rx.await.expect("recv started");
17530            // The call is now genuinely in flight. Ask it to park.
17531            interrupt.park_because("higher-priority task waiting");
17532            // Nothing the node does can observe this yet - there is no
17533            // check inside it, by construction - so let the executor run
17534            // anything pending and then let the node finish on its own.
17535            tokio::task::yield_now().await;
17536            finish_tx.send(()).expect("send finish");
17537        };
17538
17539        let (node_result, ()) = tokio::join!(node, interrupter);
17540        assert_eq!(
17541            node_result, "node finished",
17542            "the in-flight call ran to completion"
17543        );
17544
17545        // Only now, at the boundary the real `execute` would check right
17546        // after this node, does the park take effect.
17547        assert!(runner.park_here().expect("park_here"));
17548        assert!(runner.state.parked);
17549    }
17550
17551    /// A run parked mid-competition carries every field it had accumulated
17552    /// through the exact same disk round-trip an ordinary resume uses -
17553    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
17554    /// Nothing about parking for an interrupt is a special case of that path;
17555    /// this is what proves it rather than assuming it.
17556    #[test]
17557    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
17558        crate::run::pin_test_home();
17559        let mut runner = runner_at(RunStatus::Judging);
17560        // `Runner::resume` re-resolves roles from the saved config, which
17561        // refuses an empty roster - give it the same minimal one `conductor`
17562        // itself uses.
17563        runner.state.config.agents = vec![conductor()];
17564        runner.state.candidates = vec![Candidate {
17565            index: 0,
17566            label: 'A',
17567            agent: "alpha".to_owned(),
17568            branch: "magi/x/A".to_owned(),
17569            worktree: PathBuf::from("/nonexistent/worktree"),
17570            summary: "did the thing".to_owned(),
17571            stat: "1 file changed".to_owned(),
17572            files: 1,
17573            commits: 1,
17574            empty: false,
17575            failed: None,
17576            verified_noop: None,
17577            duration_ms: 1234,
17578            folded: false,
17579        }];
17580        let run_id = runner.state.id.clone();
17581
17582        let interrupt = Pause::new();
17583        runner.watch_interrupt(interrupt.clone());
17584        interrupt.park_because("task c3d4 asked to run first");
17585        assert!(runner.park_here().expect("park_here"));
17586
17587        let resumed = Runner::resume(&run_id).expect("resume");
17588        assert_eq!(resumed.state.candidates.len(), 1);
17589        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
17590        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
17591        assert_eq!(resumed.state.status, runner.state.status);
17592        assert!(
17593            resumed.state.parked,
17594            "still parked until `execute` actually walks the graph again"
17595        );
17596        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
17597    }
17598
17599    /// A fresh open question on `run`, stored and handed back for assertions.
17600    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
17601        let mut q = ask::Question::new(
17602            run.to_owned(),
17603            "implement".to_owned(),
17604            "impl-A".to_owned(),
17605            "Which storage backend should the cache use?".to_owned(),
17606            String::new(),
17607            vec!["SQLite".to_owned(), "Redis".to_owned()],
17608        );
17609        store.put(&mut q).unwrap();
17610        q
17611    }
17612
17613    #[test]
17614    fn a_failed_runs_open_question_is_abandoned() {
17615        ask_test_home();
17616        let store = ask::Questions::open();
17617        let mut runner = runner_at(RunStatus::Failed);
17618        let run = runner.state.id.clone();
17619        let q = ask_open_question(&store, &run);
17620
17621        runner.settle_questions();
17622
17623        let back = store.get(&q.id).unwrap();
17624        assert!(
17625            !back.status.open(),
17626            "the seat that asked died with the run; nobody is left to read an answer"
17627        );
17628        assert!(
17629            back.detail.contains(&run) && back.detail.contains("failed"),
17630            "the reason names what the run became, not just that it is gone: {}",
17631            back.detail
17632        );
17633    }
17634
17635    #[test]
17636    fn a_merged_runs_open_question_is_abandoned_too() {
17637        ask_test_home();
17638        let store = ask::Questions::open();
17639        // A run that finishes cleanly still leaves nobody to read an answer -
17640        // this is not only a failure-path cleanup.
17641        for status in [RunStatus::Merged, RunStatus::Ready] {
17642            let mut runner = runner_at(status);
17643            let run = runner.state.id.clone();
17644            let q = ask_open_question(&store, &run);
17645
17646            runner.settle_questions();
17647
17648            let back = store.get(&q.id).unwrap();
17649            assert!(
17650                !back.status.open(),
17651                "{status:?} run's question must not outlive the run"
17652            );
17653        }
17654    }
17655
17656    #[test]
17657    fn a_still_resumable_runs_open_question_is_left_alone() {
17658        ask_test_home();
17659        let store = ask::Questions::open();
17660        // `Blocked` and `Stalled` can still be resumed — the candidates, the
17661        // review round and the seat sessions are all still on disk — so a
17662        // question asked mid-round may yet get a real answer from a real
17663        // resume. Sweeping it here would be exactly the failure mode this
17664        // whole feature exists to avoid on the other side.
17665        for status in [RunStatus::Blocked, RunStatus::Stalled] {
17666            let mut runner = runner_at(status);
17667            let run = runner.state.id.clone();
17668            let q = ask_open_question(&store, &run);
17669
17670            runner.settle_questions();
17671
17672            let back = store.get(&q.id).unwrap();
17673            assert!(
17674                back.status.open(),
17675                "{status:?} is still alive; the question must still be waiting"
17676            );
17677        }
17678    }
17679
17680    #[test]
17681    fn settle_questions_never_touches_an_already_answered_question() {
17682        ask_test_home();
17683        let store = ask::Questions::open();
17684        let mut runner = runner_at(RunStatus::Failed);
17685        let run = runner.state.id.clone();
17686        let mut q = ask_open_question(&store, &run);
17687        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
17688            .unwrap();
17689        store.put(&mut q).unwrap();
17690
17691        // Called twice, the way a crash-recovered daemon reclaim and the
17692        // graph's own cleanup both can for the same run — `abandon_for_run`
17693        // only ever touches what is still open, so this must be inert both
17694        // times, not merely the second.
17695        runner.settle_questions();
17696        runner.settle_questions();
17697
17698        let back = store.get(&q.id).unwrap();
17699        assert_eq!(
17700            back.status,
17701            ask::QuestionStatus::Answered,
17702            "a real answer is a decision on record, never overwritten by a sweep"
17703        );
17704    }
17705
17706    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
17707    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
17708    /// without merging, whose winner is still the operator's answer to read.
17709    /// Nothing previously called `fold_run` itself with a real `tally`, so
17710    /// this is the first test to pin down the one distinction the whole
17711    /// automatic-fold feature depends on: the winner's worktree and branch
17712    /// must survive, everything else sharing the run's worktree bay - a
17713    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
17714    /// second sweep treats every non-winner directory under the bay alike -
17715    /// must not.
17716    #[tokio::test]
17717    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
17718        crate::run::pin_test_home();
17719        let tmp = tempfile::tempdir().expect("tempdir");
17720        let repo = tmp.path().join("repo");
17721        std::fs::create_dir_all(&repo).unwrap();
17722        init_repo(&repo);
17723
17724        let mut config = Config::default();
17725        config.graph.worktree_root = Some(tmp.path().join("wt"));
17726
17727        let mut state = RunState::new(
17728            repo.clone(),
17729            "main".to_owned(),
17730            "deadbeef".to_owned(),
17731            "task".to_owned(),
17732            config,
17733        );
17734        let root = state.worktree_root();
17735        let wt_a = root.join("cand-A");
17736        let wt_b = root.join("cand-B");
17737        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
17738            .await
17739            .expect("worktree A");
17740        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
17741            .await
17742            .expect("worktree B");
17743
17744        state.candidates = vec![
17745            Candidate {
17746                index: 0,
17747                label: 'A',
17748                agent: "alpha".to_owned(),
17749                branch: "magi/x/A".to_owned(),
17750                worktree: wt_a.clone(),
17751                summary: String::new(),
17752                stat: String::new(),
17753                files: 0,
17754                commits: 0,
17755                empty: false,
17756                failed: None,
17757                verified_noop: None,
17758                duration_ms: 0,
17759                folded: false,
17760            },
17761            Candidate {
17762                index: 1,
17763                label: 'B',
17764                agent: "beta".to_owned(),
17765                branch: "magi/x/B".to_owned(),
17766                worktree: wt_b.clone(),
17767                summary: String::new(),
17768                stat: String::new(),
17769                files: 0,
17770                commits: 0,
17771                empty: false,
17772                failed: None,
17773                verified_noop: None,
17774                duration_ms: 0,
17775                folded: false,
17776            },
17777        ];
17778        state.tally = Some(Tally {
17779            first_choice: BTreeMap::from([('A', 1)]),
17780            borda: BTreeMap::new(),
17781            winner: 'A',
17782            rankings: 1,
17783            unanimous_initial: true,
17784            deliberated: false,
17785            changed_votes: 0,
17786            unanimous_final: true,
17787            tie_break: None,
17788            judges: 1,
17789            present: 1,
17790            quorum: 1,
17791            met_quorum: true,
17792            uncontested: None,
17793        });
17794        state.status = RunStatus::Ready;
17795
17796        fold_run(&mut state, false, &crate::run::home())
17797            .await
17798            .expect("fold_run");
17799
17800        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
17801        assert!(
17802            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17803            "the unmerged winner's branch survives"
17804        );
17805        assert!(
17806            !state.candidates[0].folded,
17807            "the winner is not marked folded"
17808        );
17809
17810        assert!(!wt_b.exists(), "the loser's worktree is removed");
17811        assert!(
17812            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
17813            "the loser's branch is removed"
17814        );
17815        assert!(state.candidates[1].folded, "the loser is marked folded");
17816    }
17817
17818    #[tokio::test]
17819    #[cfg(unix)]
17820    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
17821        use std::os::fd::AsRawFd as _;
17822        use std::time::Duration;
17823
17824        let tmp = tempfile::tempdir().unwrap();
17825        let repo = tmp.path().join("repo");
17826        std::fs::create_dir_all(&repo).unwrap();
17827        init_repo(&repo);
17828        let common = repo.join(".git");
17829        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
17830        let home = tmp.path().join("home");
17831        let mut config = Config::default();
17832        config.coordination = crate::config::test_coordination();
17833        config.graph.worktree_root = Some(tmp.path().join("wt"));
17834        config.graph.timeout_resource_wait = 1;
17835        let mut state = RunState::new(
17836            repo.clone(),
17837            "main".into(),
17838            "deadbeef".into(),
17839            "task".into(),
17840            config,
17841        );
17842        let root = state.worktree_root();
17843        let candidate = root.join("cand-A");
17844        let reviewer = root.join("review-1");
17845        let branch = state.branch_for('A');
17846        git::worktree_add_branch(&repo, &candidate, &branch, "main")
17847            .await
17848            .unwrap();
17849        git::worktree_add_detached(&repo, &reviewer, "main")
17850            .await
17851            .unwrap();
17852        git::acquire_worktree_config(&repo).await.unwrap();
17853        state.enabled_worktree_config = true;
17854        state.status = RunStatus::Ready;
17855        state.candidates.push(Candidate {
17856            index: 0,
17857            label: 'A',
17858            agent: "implement".into(),
17859            branch: branch.clone(),
17860            worktree: candidate.clone(),
17861            summary: String::new(),
17862            stat: String::new(),
17863            files: 0,
17864            commits: 0,
17865            empty: false,
17866            failed: None,
17867            verified_noop: None,
17868            duration_ms: 0,
17869            folded: false,
17870        });
17871        state.save_under(&home).unwrap();
17872        let config_before = std::fs::read(common.join("config")).unwrap();
17873        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
17874            .await
17875            .unwrap();
17876        // A separate OS handle bypasses the launcher's local gate, exactly as
17877        // an external updater does, and must still prevent all folding work.
17878        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
17879        assert_eq!(
17880            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
17881            0
17882        );
17883        let mut fold = Box::pin(fold_run(&mut state, true, &home));
17884        assert!(
17885            tokio::time::timeout(Duration::from_millis(200), &mut fold)
17886                .await
17887                .is_err()
17888        );
17889        assert!(candidate.exists());
17890        assert!(reviewer.exists());
17891        assert!(git::branch_exists(&repo, &branch).await.unwrap());
17892        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17893        assert_eq!(
17894            git::git(&repo, &["worktree", "list", "--porcelain"])
17895                .await
17896                .unwrap(),
17897            worktrees_before
17898        );
17899        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
17900            .await
17901            .unwrap()
17902            .unwrap_err();
17903        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
17904        drop(fold);
17905        assert!(candidate.exists() && reviewer.exists());
17906        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
17907        assert_eq!(
17908            state.resource_block.as_ref().unwrap().kind,
17909            "repository coordination"
17910        );
17911        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17912        drop(external);
17913        let mut fold = Box::pin(fold_run(&mut state, true, &home));
17914        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
17915            .await
17916            .unwrap()
17917            .unwrap();
17918        drop(fold);
17919        assert!(removed.contains(&branch));
17920        assert!(!root.exists());
17921        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
17922        assert!(state.candidates[0].folded);
17923        assert!(!state.enabled_worktree_config);
17924        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
17925        let enabled = git::git_raw(
17926            &repo,
17927            &["config", "--local", "--get", "extensions.worktreeConfig"],
17928        )
17929        .await
17930        .unwrap();
17931        assert!(!enabled.ok(), "the last config reference was released");
17932    }
17933
17934    /// A branch handed to a later run is that run's (and its pull request's):
17935    /// folding the run that released it must not delete it.
17936    #[tokio::test]
17937    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
17938        let tmp = tempfile::tempdir().expect("tempdir");
17939        let repo = tmp.path().join("repo");
17940        std::fs::create_dir_all(&repo).unwrap();
17941        init_repo(&repo);
17942        let home = tmp.path().join("home");
17943
17944        let mut config = Config::default();
17945        config.graph.worktree_root = Some(tmp.path().join("wt"));
17946        let mut state = RunState::new(
17947            repo.clone(),
17948            "main".to_owned(),
17949            "deadbeef".to_owned(),
17950            "task".to_owned(),
17951            config,
17952        );
17953        // The worktree is already gone (released); the branch survives.
17954        git::git(&repo, &["branch", "magi/x/A", "main"])
17955            .await
17956            .expect("branch");
17957        state.candidates = vec![Candidate {
17958            index: 0,
17959            label: 'A',
17960            agent: "alpha".to_owned(),
17961            branch: "magi/x/A".to_owned(),
17962            worktree: state.worktree_root().join("cand-A"),
17963            summary: String::new(),
17964            stat: String::new(),
17965            files: 0,
17966            commits: 0,
17967            empty: false,
17968            failed: None,
17969            verified_noop: None,
17970            duration_ms: 0,
17971            folded: true,
17972        }];
17973        state.released_to = Some("20260901-000000-new1".to_owned());
17974        state.released_branches = vec!["magi/x/A".to_owned()];
17975
17976        fold_run(&mut state, true, &home).await.expect("fold_run");
17977        state.status = RunStatus::Merged;
17978        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
17979        state.merge = Some(crate::run::MergeOutcome {
17980            mode: MergeMode::Pr,
17981            ok: true,
17982            detail: "confirmed merged".into(),
17983            empty: false,
17984        });
17985        fold_run(&mut state, true, &home)
17986            .await
17987            .expect("merged fold");
17988
17989        assert!(
17990            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17991            "the handed-over branch survives a fold"
17992        );
17993    }
17994
17995    /// A winner with nothing ahead of the base is caught before `gh` is ever
17996    /// asked for a pull request, and the message carries what the task's
17997    /// references resolved to.
17998    #[tokio::test]
17999    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
18000        let tmp = tempfile::tempdir().expect("tempdir");
18001        let repo = tmp.path().join("repo");
18002        std::fs::create_dir_all(&repo).unwrap();
18003        init_repo(&repo);
18004        let run = |args: &[&str]| {
18005            let out = std::process::Command::new("git")
18006                .quiet()
18007                .args(args)
18008                .current_dir(&repo)
18009                .output()
18010                .expect("spawn git");
18011            assert!(out.status.success(), "git {args:?}");
18012        };
18013        run(&["branch", "magi/x/A"]);
18014        run(&["checkout", "-q", "-b", "magi/x/B"]);
18015        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
18016        run(&["add", "-A"]);
18017        run(&["commit", "-q", "-m", "work"]);
18018        run(&["checkout", "-q", "main"]);
18019
18020        // A pull request is compared against the remote's base, so the
18021        // fixture needs one. Before it exists nothing can be read, and the
18022        // answer must be "not empty".
18023        let probe = RunState::new(
18024            repo.clone(),
18025            "main".to_owned(),
18026            "deadbeef".to_owned(),
18027            "task".to_owned(),
18028            Config::default(),
18029        );
18030        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
18031        let bare = tmp.path().join("origin.git");
18032        let out = std::process::Command::new("git")
18033            .quiet()
18034            .args(["init", "-q", "--bare"])
18035            .arg(&bare)
18036            .output()
18037            .expect("spawn git");
18038        assert!(out.status.success(), "git init --bare");
18039        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
18040        run(&["push", "-q", "origin", "main"]);
18041
18042        let mut state = RunState::new(
18043            repo.clone(),
18044            "main".to_owned(),
18045            "deadbeef".to_owned(),
18046            "task".to_owned(),
18047            Config::default(),
18048        );
18049        state.seeds = vec![refs::Seed {
18050            token: "magi/27b2/A".to_owned(),
18051            kind: refs::SeedKind::Unresolved,
18052            sha: String::new(),
18053            branch: true,
18054            detail: "no branch or commit named magi/27b2/A".to_owned(),
18055        }];
18056
18057        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
18058        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
18059        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
18060        let detail = empty_candidate_detail(&state, "main");
18061        assert!(detail.starts_with("empty candidate"), "{detail}");
18062        assert!(!detail.contains("magi/27b2/A"), "{detail}");
18063        assert!(detail.contains("--start-from"), "{detail}");
18064    }
18065
18066    /// `status == Ready` used to be read as "this is the harmless
18067    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
18068    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
18069    /// run's PR was closed without merging — and reentering `merge` with
18070    /// `mode` still `Pr` does not know the difference, so it pushed and
18071    /// opened a second pull request. `mode == Local` reproduces the same
18072    /// blind spot without a network call: reentry must not attempt another
18073    /// git merge once this node has already recorded an outcome.
18074    #[tokio::test]
18075    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
18076        let tmp = tempfile::tempdir().expect("tempdir");
18077        let repo = tmp.path().join("repo");
18078        std::fs::create_dir_all(&repo).unwrap();
18079        init_repo(&repo);
18080
18081        let mut config = Config::default();
18082        config.merge.mode = MergeMode::Local;
18083
18084        let mut state = RunState::new(
18085            repo.clone(),
18086            "main".to_owned(),
18087            "deadbeef".to_owned(),
18088            "task".to_owned(),
18089            config,
18090        );
18091        state.candidates = vec![Candidate {
18092            index: 0,
18093            label: 'A',
18094            agent: "alpha".to_owned(),
18095            branch: "does-not-exist".to_owned(),
18096            worktree: repo.clone(),
18097            summary: String::new(),
18098            stat: String::new(),
18099            files: 0,
18100            commits: 0,
18101            empty: false,
18102            failed: None,
18103            verified_noop: None,
18104            duration_ms: 0,
18105            folded: false,
18106        }];
18107        state.tally = Some(Tally {
18108            first_choice: BTreeMap::from([('A', 1)]),
18109            borda: BTreeMap::new(),
18110            winner: 'A',
18111            rankings: 1,
18112            unanimous_initial: true,
18113            deliberated: false,
18114            changed_votes: 0,
18115            unanimous_final: true,
18116            tie_break: None,
18117            judges: 0,
18118            present: 0,
18119            quorum: 0,
18120            met_quorum: true,
18121            uncontested: Some("only candidate A produced a change".to_owned()),
18122        });
18123        state.reviews = vec![ReviewRound {
18124            round: 1,
18125            head: "deadbeef".to_owned(),
18126            verified_head: None,
18127            verified_at: None,
18128            reviews: Vec::new(),
18129            e2e: Vec::new(),
18130            fix: None,
18131            blocking: 0,
18132            answered: 0,
18133            expected: 0,
18134            clean: true,
18135            verify_retried: false,
18136            e2e_deferred: false,
18137            e2e_defer_reason: None,
18138            progressed: false,
18139            vote_split: false,
18140            reconsideration: Vec::new(),
18141            verdict: None,
18142        }];
18143        state.gate = vec![CommandOutcome {
18144            command: "test".to_owned(),
18145            code: Some(0),
18146            output_tail: String::new(),
18147            duration_ms: 0,
18148            resource_blocked: false,
18149        }];
18150        state.gate_ran = true;
18151        // Reached its conclusion already — e.g. `land` closing the PR without
18152        // merging it, which (like the honest `MergeMode::None` path) leaves
18153        // `status` at `Ready`. The recorded outcome is what actually marks
18154        // this node done.
18155        state.status = RunStatus::Ready;
18156        state.merge = Some(MergeOutcome {
18157            mode: MergeMode::Local,
18158            ok: false,
18159            detail: "already concluded".to_owned(),
18160            empty: false,
18161        });
18162
18163        let mut runner = Runner {
18164            state,
18165            roles: ResolvedRoles {
18166                implementers: Vec::new(),
18167                judges: Vec::new(),
18168                reviewers: Vec::new(),
18169                fixer: None,
18170                conductor: conductor(),
18171                implementer_roster: Vec::new(),
18172                judge_roster: Vec::new(),
18173                reviewer_roster: Vec::new(),
18174            },
18175            sem: Arc::new(Semaphore::new(1)),
18176            pause: Pause::new(),
18177            interrupt: Pause::new(),
18178        };
18179
18180        runner.merge().await.expect("merge");
18181
18182        assert_eq!(
18183            runner.state.status,
18184            RunStatus::Ready,
18185            "a concluded run's status must not change on reentry"
18186        );
18187        assert_eq!(
18188            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
18189            Some("already concluded"),
18190            "merge must not run again once the node already recorded an outcome"
18191        );
18192    }
18193
18194    /// `gate` leaves `state.gate_ran` false both before it has ever run and
18195    /// when its last attempt was resource-blocked (the shared build cache
18196    /// could not be acquired or confirmed fresh in time - see
18197    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
18198    /// `Vec` this also leaves behind used to read as "nothing failed" and let
18199    /// a run merge a tree the gate never actually checked - exactly the case
18200    /// a contended cache produces on every retry until it clears. `merge`
18201    /// must refuse until `gate` has actually recorded an attempt.
18202    #[tokio::test]
18203    async fn merge_refuses_a_gate_that_has_not_actually_run() {
18204        let tmp = tempfile::tempdir().expect("tempdir");
18205        let repo = tmp.path().join("repo");
18206        std::fs::create_dir_all(&repo).unwrap();
18207        init_repo(&repo);
18208
18209        let mut config = Config::default();
18210        config.merge.mode = MergeMode::Local;
18211
18212        let mut state = RunState::new(
18213            repo.clone(),
18214            "main".to_owned(),
18215            "deadbeef".to_owned(),
18216            "task".to_owned(),
18217            config,
18218        );
18219        state.candidates = vec![Candidate {
18220            index: 0,
18221            label: 'A',
18222            agent: "alpha".to_owned(),
18223            branch: "does-not-exist".to_owned(),
18224            worktree: repo.clone(),
18225            summary: String::new(),
18226            stat: String::new(),
18227            files: 0,
18228            commits: 0,
18229            empty: false,
18230            failed: None,
18231            verified_noop: None,
18232            duration_ms: 0,
18233            folded: false,
18234        }];
18235        state.tally = Some(Tally {
18236            first_choice: BTreeMap::from([('A', 1)]),
18237            borda: BTreeMap::new(),
18238            winner: 'A',
18239            rankings: 1,
18240            unanimous_initial: true,
18241            deliberated: false,
18242            changed_votes: 0,
18243            unanimous_final: true,
18244            tie_break: None,
18245            judges: 0,
18246            present: 0,
18247            quorum: 0,
18248            met_quorum: true,
18249            uncontested: Some("only candidate A produced a change".to_owned()),
18250        });
18251        state.reviews = vec![ReviewRound {
18252            round: 1,
18253            head: "deadbeef".to_owned(),
18254            verified_head: None,
18255            verified_at: None,
18256            reviews: Vec::new(),
18257            e2e: Vec::new(),
18258            fix: None,
18259            blocking: 0,
18260            answered: 0,
18261            expected: 0,
18262            clean: true,
18263            verify_retried: false,
18264            e2e_deferred: false,
18265            e2e_defer_reason: None,
18266            progressed: false,
18267            vote_split: false,
18268            reconsideration: Vec::new(),
18269            verdict: None,
18270        }];
18271        // The point: `gate` has not recorded anything yet.
18272        state.gate = Vec::new();
18273        state.gate_ran = false;
18274        state.status = RunStatus::Gating;
18275
18276        let mut runner = Runner {
18277            state,
18278            roles: ResolvedRoles {
18279                implementers: Vec::new(),
18280                judges: Vec::new(),
18281                reviewers: Vec::new(),
18282                fixer: None,
18283                conductor: conductor(),
18284                implementer_roster: Vec::new(),
18285                judge_roster: Vec::new(),
18286                reviewer_roster: Vec::new(),
18287            },
18288            sem: Arc::new(Semaphore::new(1)),
18289            pause: Pause::new(),
18290            interrupt: Pause::new(),
18291        };
18292
18293        runner.merge().await.expect("merge");
18294
18295        assert!(
18296            runner.state.merge.is_none(),
18297            "an empty gate must never be read as a passing one: {:?}",
18298            runner.state.merge
18299        );
18300    }
18301
18302    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
18303    /// commands configured and `merge.mode` is `none` (a review-only run).
18304    /// `gate` must still record a real attempt — zero commands, vacuously
18305    /// passed — rather than leaving `state.gate` empty in a way `merge`
18306    /// cannot tell apart from "never ran"; otherwise the run reaches
18307    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
18308    #[tokio::test]
18309    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
18310        let tmp = tempfile::tempdir().expect("tempdir");
18311        let repo = tmp.path().join("repo");
18312        std::fs::create_dir_all(&repo).unwrap();
18313        init_repo(&repo);
18314
18315        // Default config: `verify.gate` empty, `merge.mode` is `none`.
18316        let config = Config::default();
18317
18318        let mut state = RunState::new(
18319            repo.clone(),
18320            "main".to_owned(),
18321            "deadbeef".to_owned(),
18322            "task".to_owned(),
18323            config,
18324        );
18325        state.candidates = vec![Candidate {
18326            index: 0,
18327            label: 'A',
18328            agent: "alpha".to_owned(),
18329            branch: "does-not-exist".to_owned(),
18330            worktree: repo.clone(),
18331            summary: String::new(),
18332            stat: String::new(),
18333            files: 0,
18334            commits: 0,
18335            empty: false,
18336            failed: None,
18337            verified_noop: None,
18338            duration_ms: 0,
18339            folded: false,
18340        }];
18341        state.tally = Some(Tally {
18342            first_choice: BTreeMap::from([('A', 1)]),
18343            borda: BTreeMap::new(),
18344            winner: 'A',
18345            rankings: 1,
18346            unanimous_initial: true,
18347            deliberated: false,
18348            changed_votes: 0,
18349            unanimous_final: true,
18350            tie_break: None,
18351            judges: 0,
18352            present: 0,
18353            quorum: 0,
18354            met_quorum: true,
18355            uncontested: Some("only candidate A produced a change".to_owned()),
18356        });
18357        state.reviews = vec![ReviewRound {
18358            round: 1,
18359            head: "deadbeef".to_owned(),
18360            verified_head: None,
18361            verified_at: None,
18362            reviews: Vec::new(),
18363            e2e: Vec::new(),
18364            fix: None,
18365            blocking: 0,
18366            answered: 0,
18367            expected: 0,
18368            clean: true,
18369            verify_retried: false,
18370            e2e_deferred: false,
18371            e2e_defer_reason: None,
18372            progressed: false,
18373            vote_split: false,
18374            reconsideration: Vec::new(),
18375            verdict: None,
18376        }];
18377
18378        let mut runner = Runner {
18379            state,
18380            roles: ResolvedRoles {
18381                implementers: Vec::new(),
18382                judges: Vec::new(),
18383                reviewers: Vec::new(),
18384                fixer: None,
18385                conductor: conductor(),
18386                implementer_roster: Vec::new(),
18387                judge_roster: Vec::new(),
18388                reviewer_roster: Vec::new(),
18389            },
18390            sem: Arc::new(Semaphore::new(1)),
18391            pause: Pause::new(),
18392            interrupt: Pause::new(),
18393        };
18394
18395        runner.gate().await.expect("gate");
18396        assert!(
18397            runner.state.gate_ran,
18398            "zero configured commands is still a real attempt, not an unrun gate"
18399        );
18400        assert!(runner.state.gate.is_empty());
18401        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
18402        assert_ne!(
18403            runner.state.status,
18404            RunStatus::Blocked,
18405            "a gate with nothing to check must not read as failed"
18406        );
18407
18408        runner.merge().await.expect("merge");
18409        assert_eq!(
18410            runner.state.status,
18411            RunStatus::Ready,
18412            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
18413        );
18414    }
18415
18416    /// `Config::cache_dir` is derived from `verify.e2e` as well as
18417    /// `verify.gate` (so the e2e leg and the final gate never build against
18418    /// different directories). With zero `verify.gate` commands but a
18419    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
18420    /// that lease before discovering it had nothing to run - so a repo with
18421    /// no gate commands could come back `resource_blocked` (and therefore
18422    /// still `gate_ran == false`) on nothing but an unrelated run holding the
18423    /// cache, exactly the contention this run's own zero commands could
18424    /// never have touched. `gate` must recognise there is nothing to check
18425    /// before it ever asks for the lease.
18426    #[tokio::test]
18427    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
18428        crate::run::pin_test_home();
18429        let home = crate::run::home();
18430
18431        let tmp = tempfile::tempdir().expect("tempdir");
18432        let repo = tmp.path().join("repo");
18433        std::fs::create_dir_all(&repo).unwrap();
18434        init_repo(&repo);
18435        // Unique to this test, so holding its lease cannot collide with
18436        // another test sharing the same process-wide `home`.
18437        let cache_dir = tmp.path().join("target");
18438
18439        let mut config = Config::default();
18440        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
18441        // `verify.gate` stays empty (the default). Any accidental cache
18442        // acquisition must report contention immediately, regardless of how
18443        // long unrelated Git subprocesses take under parallel test load.
18444        config.graph.timeout_verify = Some(2);
18445        config.graph.timeout_resource_wait = 0;
18446
18447        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18448        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18449            .expect("no io error acquiring directly")
18450        {
18451            crate::cache::AcquireOutcome::Acquired(g) => g,
18452            crate::cache::AcquireOutcome::Busy(b) => {
18453                panic!("expected the direct acquire to win the lease first: {b:?}")
18454            }
18455        };
18456
18457        let mut state = RunState::new(
18458            repo.clone(),
18459            "main".to_owned(),
18460            "deadbeef".to_owned(),
18461            "task".to_owned(),
18462            config,
18463        );
18464        state.candidates = vec![Candidate {
18465            index: 0,
18466            label: 'A',
18467            agent: "alpha".to_owned(),
18468            branch: "does-not-exist".to_owned(),
18469            worktree: repo.clone(),
18470            summary: String::new(),
18471            stat: String::new(),
18472            files: 0,
18473            commits: 0,
18474            empty: false,
18475            failed: None,
18476            verified_noop: None,
18477            duration_ms: 0,
18478            folded: false,
18479        }];
18480        state.tally = Some(Tally {
18481            first_choice: BTreeMap::from([('A', 1)]),
18482            borda: BTreeMap::new(),
18483            winner: 'A',
18484            rankings: 1,
18485            unanimous_initial: true,
18486            deliberated: false,
18487            changed_votes: 0,
18488            unanimous_final: true,
18489            tie_break: None,
18490            judges: 0,
18491            present: 0,
18492            quorum: 0,
18493            met_quorum: true,
18494            uncontested: Some("only candidate A produced a change".to_owned()),
18495        });
18496        state.reviews = vec![ReviewRound {
18497            round: 1,
18498            head: "deadbeef".to_owned(),
18499            verified_head: None,
18500            verified_at: None,
18501            reviews: Vec::new(),
18502            e2e: Vec::new(),
18503            fix: None,
18504            blocking: 0,
18505            answered: 0,
18506            expected: 0,
18507            clean: true,
18508            verify_retried: false,
18509            e2e_deferred: false,
18510            e2e_defer_reason: None,
18511            progressed: false,
18512            vote_split: false,
18513            reconsideration: Vec::new(),
18514            verdict: None,
18515        }];
18516
18517        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
18518        state.reviews[0].e2e = vec![CommandOutcome {
18519            command: state.config.verify.e2e[0].clone(),
18520            code: Some(0),
18521            output_tail: String::new(),
18522            duration_ms: 0,
18523            resource_blocked: false,
18524        }];
18525        let mut runner = Runner {
18526            state,
18527            roles: ResolvedRoles {
18528                implementers: Vec::new(),
18529                judges: Vec::new(),
18530                reviewers: Vec::new(),
18531                fixer: None,
18532                conductor: conductor(),
18533                implementer_roster: Vec::new(),
18534                judge_roster: Vec::new(),
18535                reviewer_roster: Vec::new(),
18536            },
18537            sem: Arc::new(Semaphore::new(1)),
18538            pause: Pause::new(),
18539            interrupt: Pause::new(),
18540        };
18541
18542        runner.gate().await.expect("gate");
18543        assert!(runner.state.resource_block.is_none());
18544        assert!(
18545            !runner
18546                .state
18547                .events
18548                .iter()
18549                .any(|event| { event.message.contains("waiting for the shared build cache") }),
18550            "a gate with nothing to run must never request the held cache lease"
18551        );
18552        assert!(
18553            runner.state.gate_ran,
18554            "zero commands is still a real, immediate attempt"
18555        );
18556        assert!(runner.state.gate.is_empty());
18557        assert_ne!(
18558            runner.state.status,
18559            RunStatus::Blocked,
18560            "must not read as resource-blocked on a lease it never asked for"
18561        );
18562    }
18563
18564    /// A separate process carries a live worker's environment throughout all
18565    /// verification entry points; the parallel harness never changes its env.
18566    #[cfg(unix)]
18567    #[tokio::test]
18568    async fn verification_is_standalone_inside_a_live_worker() {
18569        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
18570        if std::env::var_os(CHILD).is_none() {
18571            let home = tempfile::tempdir().unwrap();
18572            let generation = crate::worker::pin_current(home.path()).unwrap();
18573            let output = std::process::Command::new(std::env::current_exe().unwrap())
18574                .args([
18575                    "--exact",
18576                    "graph::tests::verification_is_standalone_inside_a_live_worker",
18577                    "--nocapture",
18578                ])
18579                .env(CHILD, "1")
18580                .env("MAGI_HOME", home.path())
18581                .env("MAGI_WORKER_HOME", home.path())
18582                .env("MAGI_WORKER_ID", "abcdef")
18583                .env("MAGI_WORKER_GENERATION", &generation.id)
18584                .env("MAGI_WORKER_EXE", &generation.executable)
18585                .env("MAGI_RUN", "verification-parent")
18586                .env("MAGI_NODE", "gate")
18587                .env("MAGI_SEAT", "worker-fixture")
18588                .env("MAGI_PROCESS_OWNER", "fixture-owner")
18589                .env("MAGI_NO_AUTOUPDATE", "0")
18590                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
18591                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
18592                .env(
18593                    "VERIFY_EXPECT_TOOLCHAIN",
18594                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
18595                )
18596                .env(
18597                    "VERIFY_EXPECT_CACHE",
18598                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
18599                )
18600                .output()
18601                .unwrap();
18602            assert!(
18603                output.status.success()
18604                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
18605                "{}\n{}",
18606                String::from_utf8_lossy(&output.stdout),
18607                String::from_utf8_lossy(&output.stderr)
18608            );
18609            return;
18610        }
18611        let home = crate::run::home();
18612        let directory = home.join("workers/jobs/abcdef");
18613        std::fs::create_dir_all(&directory).unwrap();
18614        let job = crate::worker::Job {
18615            id: "abcdef".into(),
18616            sequence: 1,
18617            kind: "task".into(),
18618            key: "task:verification".into(),
18619            payload: serde_json::json!({}),
18620            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
18621            version: Some(env!("CARGO_PKG_VERSION").into()),
18622            status: crate::worker::Status::Running,
18623            result: None,
18624            pid: Some(std::process::id()),
18625            created_at: 0,
18626            finished_at: None,
18627        };
18628        std::fs::write(
18629            directory.join("job.json"),
18630            serde_json::to_vec(&job).unwrap(),
18631        )
18632        .unwrap();
18633        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
18634        let assert_worker = || {
18635            assert_eq!(
18636                crate::worker::current_job(&home).unwrap().unwrap().id,
18637                job.id
18638            );
18639            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
18640            assert_eq!(
18641                crate::worker::helper_executable(&home).unwrap(),
18642                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
18643            );
18644            assert!(
18645                crate::updater::Checker::new(&crate::config::Update {
18646                    mode: crate::config::UpdateMode::Notify,
18647                    interval: None,
18648                    auto_restart: false,
18649                })
18650                .is_none(),
18651                "the driver remains pinned, even while tests are standalone"
18652            );
18653        };
18654        assert_worker();
18655        let repo = home.join("repo");
18656        std::fs::create_dir_all(&repo).unwrap();
18657        init_repo(&repo);
18658        // Keep storage, seat/control context, containment and tool lookup.
18659        // No inherited values (or credentials) are printed on success.
18660        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
18661            test -z "${MAGI_WORKER_GENERATION+x}" &&
18662            test -z "${MAGI_WORKER_HOME+x}" &&
18663            test -z "${MAGI_WORKER_EXE+x}" &&
18664            test "$MAGI_HOME/repo" -ef . &&
18665            test "$MAGI_RUN" = verification-parent &&
18666            test "$MAGI_NODE" = gate &&
18667            test "$MAGI_SEAT" = worker-fixture &&
18668            test -n "$MAGI_PROCESS_OWNER" &&
18669            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
18670            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
18671            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
18672            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
18673        // Negative control: an ordinary child inherits the contamination.
18674        assert!(
18675            !std::process::Command::new("sh")
18676                .args(["-c", probe])
18677                .current_dir(&repo)
18678                .status()
18679                .unwrap()
18680                .success()
18681        );
18682        let unrelated_home = tempfile::tempdir().unwrap();
18683        assert!(
18684            crate::worker::current_job(unrelated_home.path()).is_err(),
18685            "an inherited worker id cannot be resolved in a unit fixture's home"
18686        );
18687        let mut runner = runner_at(RunStatus::Gating);
18688        runner.state.repo = repo.clone();
18689        runner.state.config.disk.min_free_bytes = 0;
18690        runner.state.config.coordination = crate::config::test_coordination();
18691        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
18692        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
18693        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
18694        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
18695        let winner = Candidate {
18696            index: 0,
18697            label: 'A',
18698            agent: "fixture".into(),
18699            branch: "main".into(),
18700            worktree: repo.clone(),
18701            summary: String::new(),
18702            stat: String::new(),
18703            files: 0,
18704            commits: 0,
18705            empty: false,
18706            failed: None,
18707            verified_noop: None,
18708            duration_ms: 0,
18709            folded: false,
18710        };
18711        let (e2e, retried, pids) = run_e2e_with_retry(
18712            &mut runner.state,
18713            &["sh".into(), "-c".into()],
18714            &[format!("{probe} && printf initial")],
18715            &repo,
18716            Duration::from_secs(10),
18717            "initial",
18718        )
18719        .await;
18720        assert!(!retried && pids.is_empty());
18721        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
18722        assert_eq!(e2e[0].output_tail, "initial");
18723        let gate = runner.run_gate(&winner).await.unwrap();
18724        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
18725        // An interrupted repair must recheck e2e from saved state. Landing
18726        // reuses that state and must run both pre-gate and gate again.
18727        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
18728            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
18729            passed: false,
18730            outcomes: Vec::new(),
18731        });
18732        runner.state.save().unwrap();
18733        let mut resumed = RunState::load(&runner.state.id).unwrap();
18734        let wait = crate::repo_lock::WaitPolicy::default();
18735        assert!(matches!(
18736            check_gate_repair_e2e(&mut resumed, &winner, &wait)
18737                .await
18738                .unwrap(),
18739            GateFix::Retry
18740        ));
18741        let saved = RunState::load(&resumed.id).unwrap();
18742        let repair = saved.gate_repair_e2e.unwrap();
18743        assert!(repair.passed);
18744        assert_eq!(repair.outcomes.len(), 1);
18745        assert!(repair.outcomes[0].ok());
18746        let landing = run_landing_gate(&mut resumed, &winner, &wait)
18747            .await
18748            .unwrap();
18749        assert_eq!(landing.len(), 1);
18750        assert!(landing[0].ok(), "{landing:?}");
18751        assert_eq!(
18752            std::fs::read_to_string(repo.join("phases")).unwrap(),
18753            "pre\ngate\ne2e\npre\ngate\n"
18754        );
18755        assert_worker();
18756    }
18757
18758    #[cfg(unix)]
18759    #[tokio::test]
18760    async fn verification_keeps_the_panic_and_the_full_output() {
18761        crate::run::pin_test_home();
18762        let tmp = tempfile::tempdir().unwrap();
18763        let mut config = Config::default();
18764        config.disk.min_free_bytes = 0;
18765        let mut state = RunState::new(
18766            tmp.path().to_owned(),
18767            "main".into(),
18768            "base".into(),
18769            "verification evidence".into(),
18770            config,
18771        );
18772        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
18773                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
18774                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
18775        let mut names = Vec::new();
18776        for _ in 0..2 {
18777            let (outcomes, _) = run_commands(
18778                &mut state,
18779                "gate",
18780                "gate",
18781                0,
18782                &["sh".into(), "-c".into()],
18783                &[script.into()],
18784                tmp.path(),
18785                Duration::from_secs(60),
18786            )
18787            .await;
18788            let tail = &outcomes[0].output_tail;
18789            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
18790            assert!(tail.contains("boom assertion"));
18791            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
18792            let line = tail.lines().next().unwrap();
18793            names.push(line.to_owned());
18794        }
18795        assert_ne!(names[0], names[1], "the second run must not overwrite");
18796        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
18797        assert!(full.len() > 500_000 && full.contains("boom assertion"));
18798        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
18799
18800        // Storage failures are evidence too; preserve the command's verdict.
18801        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
18802        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
18803        let (outcomes, _) = run_commands(
18804            &mut state,
18805            "gate",
18806            "gate",
18807            0,
18808            &["sh".into(), "-c".into()],
18809            &["echo failure-output; exit 7".into()],
18810            tmp.path(),
18811            Duration::from_secs(10),
18812        )
18813        .await;
18814        assert_eq!(outcomes[0].code, Some(7));
18815        assert!(state.events.iter().any(|e| {
18816            e.node == "gate" && e.message.contains("could not save the full gate output:")
18817        }));
18818    }
18819
18820    #[cfg(unix)]
18821    #[tokio::test]
18822    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
18823        crate::run::pin_test_home();
18824        let tmp = tempfile::tempdir().unwrap();
18825        let mut config = Config::default();
18826        config.disk.min_free_bytes = 0;
18827        let mut state = RunState::new(
18828            tmp.path().to_owned(),
18829            "main".into(),
18830            "base".into(),
18831            "verification evidence".into(),
18832            config,
18833        );
18834        let (outcomes, pids) = run_commands(
18835            &mut state,
18836            "gate",
18837            "gate",
18838            1,
18839            &["sh".into(), "-c".into()],
18840            &[
18841                "printf before-timeout; exec sleep 30".into(),
18842                "kill -TERM $$".into(),
18843                "exit 7".into(),
18844                "printf success".into(),
18845            ],
18846            tmp.path(),
18847            Duration::from_secs(1),
18848        )
18849        .await;
18850        wait_for_timed_out_children_to_die(&pids).await;
18851        assert_eq!(
18852            pids.len(),
18853            1,
18854            "unleased timeout retains its death-check PID"
18855        );
18856        assert_eq!(outcomes.len(), 4);
18857        assert_eq!(outcomes[0].code, None);
18858        assert!(outcomes[0].output_tail.contains("before-timeout"));
18859        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
18860        assert_eq!(outcomes[1].code, None);
18861        assert!(outcomes[1].output_tail.contains("command terminated:"));
18862        assert!(!outcomes[1].output_tail.contains("timed out"));
18863        assert_eq!(outcomes[2].code, Some(7));
18864        assert!(!outcomes[2].output_tail.contains("timed out"));
18865        assert_eq!(outcomes[3].code, Some(0));
18866        assert_eq!(outcomes[3].output_tail, "success");
18867        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
18868    }
18869
18870    /// The addendum's second gap: a `verify.gate` command running for real
18871    /// wall-clock time had nothing at all to show for it in `active` before
18872    /// `run_commands` learned to record it — a run could sit in `Gating` for
18873    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
18874    /// what was actually happening. Proven with a genuinely still-running
18875    /// command, not just a before/after check on the final state: a poller
18876    /// task reads the same `run.json` `gate()` is writing, the same way the
18877    /// phone or `magi show` would, while the shell command is still blocked
18878    /// on its own release marker.
18879    #[tokio::test]
18880    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
18881        crate::run::pin_test_home();
18882
18883        let tmp = tempfile::tempdir().expect("tempdir");
18884        let repo = tmp.path().join("repo");
18885        std::fs::create_dir_all(&repo).unwrap();
18886        init_repo(&repo);
18887
18888        let mut config = Config::default();
18889        config.verify.gate = vec![
18890            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
18891             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
18892                .to_owned(),
18893        ];
18894
18895        let mut state = RunState::new(
18896            repo.clone(),
18897            "main".to_owned(),
18898            "deadbeef".to_owned(),
18899            "task".to_owned(),
18900            config,
18901        );
18902        let run_id = state.id.clone();
18903        state.candidates = vec![Candidate {
18904            index: 0,
18905            label: 'A',
18906            agent: "alpha".to_owned(),
18907            branch: "does-not-exist".to_owned(),
18908            worktree: repo.clone(),
18909            summary: String::new(),
18910            stat: String::new(),
18911            files: 0,
18912            commits: 0,
18913            empty: false,
18914            failed: None,
18915            verified_noop: None,
18916            duration_ms: 0,
18917            folded: false,
18918        }];
18919        state.tally = Some(Tally {
18920            first_choice: BTreeMap::from([('A', 1)]),
18921            borda: BTreeMap::new(),
18922            winner: 'A',
18923            rankings: 1,
18924            unanimous_initial: true,
18925            deliberated: false,
18926            changed_votes: 0,
18927            unanimous_final: true,
18928            tie_break: None,
18929            judges: 0,
18930            present: 0,
18931            quorum: 0,
18932            met_quorum: true,
18933            uncontested: Some("only candidate A produced a change".to_owned()),
18934        });
18935        state.reviews = vec![ReviewRound {
18936            round: 1,
18937            head: "deadbeef".to_owned(),
18938            verified_head: None,
18939            verified_at: None,
18940            reviews: Vec::new(),
18941            e2e: Vec::new(),
18942            fix: None,
18943            blocking: 0,
18944            answered: 0,
18945            expected: 0,
18946            clean: true,
18947            verify_retried: false,
18948            e2e_deferred: false,
18949            e2e_defer_reason: None,
18950            progressed: false,
18951            vote_split: false,
18952            reconsideration: Vec::new(),
18953            verdict: None,
18954        }];
18955
18956        let mut runner = Runner {
18957            state,
18958            roles: ResolvedRoles {
18959                implementers: Vec::new(),
18960                judges: Vec::new(),
18961                reviewers: Vec::new(),
18962                fixer: None,
18963                conductor: conductor(),
18964                implementer_roster: Vec::new(),
18965                judge_roster: Vec::new(),
18966                reviewer_roster: Vec::new(),
18967            },
18968            sem: Arc::new(Semaphore::new(1)),
18969            pause: Pause::new(),
18970            interrupt: Pause::new(),
18971        };
18972
18973        let started_marker = repo.join("started.marker");
18974        let release_marker = repo.join("release.marker");
18975        let poller = tokio::spawn(async move {
18976            // Bounded so a regression that never records the task entry
18977            // fails this test in seconds instead of hanging the suite —
18978            // the same shape `a_park_requested_while_a_seat_is_mid_call_
18979            // does_not_cut_it_short` uses for the same reason.
18980            for _ in 0..100 {
18981                if started_marker.exists()
18982                    && let Ok(s) = crate::run::RunState::load(&run_id)
18983                    && let Some(a) = s.active.get("gate")
18984                {
18985                    std::fs::write(&release_marker, b"go").expect("release marker");
18986                    return Some(a.clone());
18987                }
18988                tokio::time::sleep(Duration::from_millis(50)).await;
18989            }
18990            None
18991        });
18992
18993        runner.gate().await.expect("gate");
18994        let captured = poller.await.expect("poller task");
18995        let captured = captured.expect(
18996            "the poller never saw a `gate` task entry in run.json while the command was \
18997             still blocked on its own release marker",
18998        );
18999
19000        assert_eq!(captured.task.as_deref(), Some("gate"));
19001        assert_eq!(captured.node, "gate");
19002        assert_eq!(captured.index, Some(1));
19003        assert_eq!(captured.total, Some(1));
19004        assert!(
19005            captured
19006                .command
19007                .as_deref()
19008                .is_some_and(|c| c.contains("started.marker")),
19009            "{captured:?}"
19010        );
19011
19012        assert!(
19013            runner.state.active.is_empty(),
19014            "the entry must be cleared once the command actually finished: {:?}",
19015            runner.state.active
19016        );
19017        assert!(runner.state.gate_ran);
19018        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
19019    }
19020
19021    /// The hand-off over a blocking finding a reviewer rejected on leaves a
19022    /// record for `land`; one with only a Minor, or no reject, leaves none.
19023    #[tokio::test]
19024    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
19025        use crate::verdict::{Finding, ReviewVote, Severity};
19026        crate::run::pin_test_home();
19027        let tmp = tempfile::tempdir().expect("tempdir");
19028        let repo = tmp.path().join("repo");
19029        std::fs::create_dir_all(&repo).unwrap();
19030        init_repo(&repo);
19031
19032        for (severity, vote, expect) in [
19033            (Severity::Major, ReviewVote::Reject, true),
19034            (Severity::Minor, ReviewVote::Reject, false),
19035            (Severity::Major, ReviewVote::Approve, false),
19036        ] {
19037            let mut round = review_round(false, 1, 1, 1, false, true);
19038            round.reviews = vec![ReviewRecord {
19039                reviewer: 1,
19040                agent: "alpha".to_owned(),
19041                summary: String::new(),
19042                findings: vec![Finding {
19043                    id: "R1-1-1".to_owned(),
19044                    severity,
19045                    file: None,
19046                    line: None,
19047                    title: "t".to_owned(),
19048                    detail: String::new(),
19049                }],
19050                vote: Some(vote),
19051                failed: None,
19052                duration_ms: 0,
19053                attempts: 0,
19054            }];
19055            let mut state = RunState::new(
19056                repo.clone(),
19057                "main".to_owned(),
19058                "deadbeef".to_owned(),
19059                "task".to_owned(),
19060                Config::default(),
19061            );
19062            state.reviews = vec![round];
19063            let mut runner = Runner {
19064                state,
19065                roles: ResolvedRoles {
19066                    implementers: Vec::new(),
19067                    judges: Vec::new(),
19068                    reviewers: Vec::new(),
19069                    fixer: None,
19070                    conductor: conductor(),
19071                    implementer_roster: Vec::new(),
19072                    judge_roster: Vec::new(),
19073                    reviewer_roster: Vec::new(),
19074                },
19075                sem: Arc::new(Semaphore::new(1)),
19076                pause: Pause::new(),
19077                interrupt: Pause::new(),
19078            };
19079            let shell = runner.state.config.shell();
19080            runner
19081                .stop_reviewing("round budget spent", &shell, &repo)
19082                .await
19083                .expect("stop_reviewing");
19084            assert_eq!(runner.state.status, RunStatus::Gating);
19085            assert_eq!(
19086                runner.state.contested_handoff.is_some(),
19087                expect,
19088                "{severity:?} + {vote:?}"
19089            );
19090        }
19091    }
19092
19093    /// The shape the incident this whole fix responds to actually had: the
19094    /// round budget spent, the last round's own e2e blocked on the shared
19095    /// build cache (held here by a live pid — this test process — exactly
19096    /// `cache`'s own unit tests' pattern for "another owner, still alive"
19097    /// without forking a process). `stop_reviewing` must retry it — not
19098    /// silently leave the round looking untouched (the catch-up-only half of
19099    /// the bug), and not read the contention as a red `e2e` and block the
19100    /// run on it (the other half). Called directly, the same way
19101    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
19102    /// above exercises `gate`, so this never needs a real cargo build to
19103    /// reach: the lease is never released, so `with_cache_lease` never gets
19104    /// past acquiring it into anything that would need a real workspace.
19105    #[tokio::test]
19106    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
19107        crate::run::pin_test_home();
19108        let home = crate::run::home();
19109
19110        let tmp = tempfile::tempdir().expect("tempdir");
19111        let repo = tmp.path().join("repo");
19112        std::fs::create_dir_all(&repo).unwrap();
19113        init_repo(&repo);
19114        let head = crate::git::rev_parse(&repo, "HEAD")
19115            .await
19116            .expect("rev-parse");
19117        // Unique to this test, so holding its lease cannot collide with
19118        // another test sharing the same process-wide `home`.
19119        let cache_dir = tmp.path().join("target");
19120
19121        let mut config = Config::default();
19122        config.verify.e2e = vec![format!(
19123            "CARGO_TARGET_DIR='{}' test -f README.md",
19124            cache_dir.display()
19125        )];
19126        config.graph.review_rounds = 1;
19127        // Bounded so a regression that does start waiting fails the test in
19128        // seconds, not hangs it.
19129        config.graph.timeout_verify = Some(2);
19130        config.graph.timeout_resource_wait = 2;
19131
19132        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19133        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19134            .expect("no io error acquiring directly")
19135        {
19136            crate::cache::AcquireOutcome::Acquired(g) => g,
19137            crate::cache::AcquireOutcome::Busy(b) => {
19138                panic!("expected the direct acquire to win the lease first: {b:?}")
19139            }
19140        };
19141
19142        let mut state = RunState::new(
19143            repo.clone(),
19144            "main".to_owned(),
19145            head.clone(),
19146            "task".to_owned(),
19147            config,
19148        );
19149        state.candidates = vec![Candidate {
19150            index: 0,
19151            label: 'A',
19152            agent: "alpha".to_owned(),
19153            branch: "does-not-exist".to_owned(),
19154            worktree: repo.clone(),
19155            summary: String::new(),
19156            stat: String::new(),
19157            files: 0,
19158            commits: 0,
19159            empty: false,
19160            failed: None,
19161            verified_noop: None,
19162            duration_ms: 0,
19163            folded: false,
19164        }];
19165        state.tally = Some(Tally {
19166            first_choice: BTreeMap::from([('A', 1)]),
19167            borda: BTreeMap::new(),
19168            winner: 'A',
19169            rankings: 1,
19170            unanimous_initial: true,
19171            deliberated: false,
19172            changed_votes: 0,
19173            unanimous_final: true,
19174            tie_break: None,
19175            judges: 0,
19176            present: 0,
19177            quorum: 0,
19178            met_quorum: true,
19179            uncontested: Some("only candidate A produced a change".to_owned()),
19180        });
19181        // The round budget's last round, deferred: `needs_catchup_run`'s
19182        // other trigger. `stop_reviewing`'s retry machinery must treat this
19183        // exactly like a resource-blocked attempt once it actually runs.
19184        state.reviews = vec![ReviewRound {
19185            round: 1,
19186            head: head.clone(),
19187            verified_head: None,
19188            verified_at: None,
19189            reviews: Vec::new(),
19190            e2e: Vec::new(),
19191            fix: None,
19192            blocking: 1,
19193            answered: 1,
19194            expected: 1,
19195            clean: false,
19196            verify_retried: false,
19197            e2e_deferred: true,
19198            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
19199            progressed: false,
19200            vote_split: false,
19201            reconsideration: Vec::new(),
19202            verdict: None,
19203        }];
19204
19205        let mut runner = Runner {
19206            state,
19207            roles: ResolvedRoles {
19208                implementers: Vec::new(),
19209                judges: Vec::new(),
19210                reviewers: Vec::new(),
19211                fixer: None,
19212                conductor: conductor(),
19213                implementer_roster: Vec::new(),
19214                judge_roster: Vec::new(),
19215                reviewer_roster: Vec::new(),
19216            },
19217            sem: Arc::new(Semaphore::new(1)),
19218            pause: Pause::new(),
19219            interrupt: Pause::new(),
19220        };
19221
19222        let shell = runner.state.config.shell();
19223        runner
19224            .stop_reviewing("round budget spent", &shell, &repo)
19225            .await
19226            .expect("stop_reviewing");
19227
19228        let last = runner.state.reviews.last().expect("round record");
19229        assert_eq!(
19230            last.e2e_status(),
19231            E2eStatus::ResourceBlocked,
19232            "the shared cache is still held; the attempt must read as blocked, not deferred or \
19233             failed: {last:?}"
19234        );
19235        assert_eq!(
19236            last.verified_head.as_deref(),
19237            Some(head.as_str()),
19238            "which commit this attempt targeted is known even though nothing finished checking \
19239             it"
19240        );
19241        let first_attempt_at = last
19242            .verified_at
19243            .expect("when this attempt ran is known too");
19244        assert_ne!(
19245            runner.state.status,
19246            RunStatus::Blocked,
19247            "contention is evidence about the machine, not the patch — it must not settle the \
19248             run as blocked: {:?}",
19249            runner.state.status
19250        );
19251        assert!(
19252            !runner
19253                .state
19254                .events
19255                .iter()
19256                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
19257            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
19258            runner.state.events
19259        );
19260
19261        // The cache is still held: a later reentry must retry the same
19262        // round's verification again — not leave it looking exactly as
19263        // untouched as the first blocked attempt, which is indistinguishable
19264        // from never having tried again at all.
19265        runner
19266            .stop_reviewing("round budget spent", &shell, &repo)
19267            .await
19268            .expect("stop_reviewing retry");
19269        assert_eq!(
19270            runner.state.reviews.len(),
19271            1,
19272            "no new round was started: {:?}",
19273            runner.state.reviews
19274        );
19275        let last = runner.state.reviews.last().expect("round record");
19276        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
19277        assert!(
19278            last.verified_at.expect("still known") > first_attempt_at,
19279            "a second reentry must be a fresh attempt, not a stale copy of the first"
19280        );
19281        assert_ne!(runner.state.status, RunStatus::Blocked);
19282
19283        held.release();
19284    }
19285
19286    #[cfg(unix)]
19287    #[tokio::test]
19288    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
19289        ask_test_home();
19290        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
19291            let tmp = tempfile::tempdir().unwrap();
19292            let repo = tmp.path().join("repo");
19293            std::fs::create_dir(&repo).unwrap();
19294            init_repo(&repo);
19295            let run_git = |args: &[&str]| {
19296                let out = std::process::Command::new("git")
19297                    .args(args)
19298                    .current_dir(&repo)
19299                    .output()
19300                    .unwrap();
19301                assert!(
19302                    out.status.success(),
19303                    "{args:?}: {}",
19304                    String::from_utf8_lossy(&out.stderr)
19305                );
19306            };
19307            let origin = tmp.path().join("origin.git");
19308            run_git(&["init", "--bare", origin.to_str().unwrap()]);
19309            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
19310            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
19311            run_git(&["add", "spec.rs"]);
19312            run_git(&["commit", "-m", "initial spec"]);
19313            run_git(&["push", "origin", "main"]);
19314            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
19315            run_git(&["checkout", "-b", "fix/review"]);
19316            std::fs::write(
19317                repo.join("check.rs"),
19318                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
19319            )
19320            .unwrap();
19321            run_git(&["add", "check.rs"]);
19322            run_git(&["commit", "-m", "use spec"]);
19323            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
19324            run_git(&["checkout", "main"]);
19325            std::fs::write(
19326                repo.join("spec.rs"),
19327                "struct AgentSpec { max_turn_secs: u64 }\n",
19328            )
19329            .unwrap();
19330            run_git(&["add", "spec.rs"]);
19331            run_git(&["commit", "-m", "extend spec"]);
19332            run_git(&["push", "origin", "main"]);
19333            run_git(&["checkout", "fix/review"]);
19334
19335            let mut runner = runner_at(RunStatus::Reviewing);
19336            runner.state.repo = repo.clone();
19337            runner.state.base_commit = base;
19338            runner.state.pr_head = Some("fix/review".into());
19339            runner.state.config.graph.review_rounds = max_rounds;
19340            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
19341            runner.state.config.graph.sessions = false;
19342            runner.state.config.disk.min_free_bytes = 0;
19343            runner.state.config.verify.e2e =
19344                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
19345            runner.state.candidates = vec![Candidate {
19346                index: 0,
19347                label: 'A',
19348                agent: "fixture".into(),
19349                branch: "fix/review".into(),
19350                worktree: repo.clone(),
19351                summary: String::new(),
19352                stat: String::new(),
19353                files: 1,
19354                commits: 1,
19355                empty: false,
19356                failed: None,
19357                verified_noop: None,
19358                duration_ms: 0,
19359                folded: false,
19360            }];
19361            runner.state.tally = Some(Tally {
19362                first_choice: BTreeMap::new(),
19363                borda: BTreeMap::new(),
19364                winner: 'A',
19365                rankings: 0,
19366                unanimous_initial: true,
19367                deliberated: false,
19368                changed_votes: 0,
19369                unanimous_final: true,
19370                tie_break: None,
19371                judges: 0,
19372                present: 0,
19373                quorum: 0,
19374                met_quorum: true,
19375                uncontested: Some("review-only".into()),
19376            });
19377            let mut last = review_round(false, 0, 1, 1, false, false);
19378            last.head = reviewed.clone();
19379            last.e2e_deferred = true;
19380            last.e2e[0].resource_blocked = true;
19381            last.e2e[0].output_tail = "waiting for the shared build cache".into();
19382            if max_rounds == 1 {
19383                last.e2e.clear();
19384            }
19385            runner.state.reviews = vec![last];
19386            let calls = tmp.path().join("fix-calls");
19387            let mut fixer = spec("repair");
19388            fixer
19389                .env
19390                .insert("FIX_CALLS".into(), calls.display().to_string());
19391            fixer.command = vec!["sh".into(), "-c".into(), r#"
19392                echo fix >> "$FIX_CALLS"
19393                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
19394                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
19395                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
19396                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
19397            "#.into()];
19398            let mut reviewer = spec("reviewer");
19399            reviewer.command = vec![
19400                "sh".into(),
19401                "-c".into(),
19402                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
19403                    .into(),
19404            ];
19405            runner.roles.fixer = Some(vec![fixer]);
19406            runner.roles.reviewers = vec![reviewer];
19407
19408            runner.sync_to_base().await.unwrap();
19409            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
19410            assert_ne!(rebased, reviewed);
19411            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
19412            if reentry {
19413                // Simulate a crash after catch-up evidence was saved but before
19414                // dispatching the fixer. No dedicated pending-repair field exists.
19415                let shell = runner.state.config.shell();
19416                runner
19417                    .stop_unresolved_review("deferred verification", &shell, &repo)
19418                    .await
19419                    .unwrap();
19420                assert!(runner.catchup_repairable());
19421                assert!(!calls.exists());
19422                runner.state = RunState::load(&runner.state.id).unwrap();
19423                assert!(runner.catchup_repairable());
19424            }
19425            runner.review_loop().await.unwrap();
19426            let first = &runner.state.reviews[0];
19427            assert_eq!(first.head, reviewed);
19428            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
19429            assert!(
19430                first.verify_retried,
19431                "a reproduced build failure gets exactly one retry: {:?}",
19432                first.e2e
19433            );
19434            assert!(first.e2e[0].output_tail.contains("E0063"));
19435            assert!(!runner.catchup_repairable());
19436            if max_rounds == 2 {
19437                assert_eq!(
19438                    runner.state.status,
19439                    RunStatus::Gating,
19440                    "{}",
19441                    crate::report::run(&runner.state)
19442                );
19443                assert_eq!(runner.state.reviews.len(), 2);
19444                assert!(first.fix.as_ref().unwrap().committed);
19445                assert!(runner.state.reviews[1].clean);
19446                assert_eq!(
19447                    runner
19448                        .state
19449                        .reviews
19450                        .iter()
19451                        .filter(|r| r.fix.is_some())
19452                        .count(),
19453                    1
19454                );
19455                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
19456                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
19457            } else {
19458                assert_eq!(runner.state.status, RunStatus::Blocked);
19459                assert!(first.fix.is_none());
19460                assert!(!calls.exists());
19461                let recovery = crate::recovery::assess(
19462                    "prior",
19463                    Some(&runner.state),
19464                    &runner.landing_base(),
19465                    crate::recovery::Relation::Diverged,
19466                );
19467                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
19468                let reasons = recovery.reasons.join("\n");
19469                assert!(
19470                    reasons.contains(&reviewed) && reasons.contains(&rebased),
19471                    "{reasons}"
19472                );
19473                assert!(
19474                    reasons.contains("base sync to")
19475                        && reasons.contains("additional review is required"),
19476                    "{reasons}"
19477                );
19478            }
19479        }
19480    }
19481
19482    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
19483    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
19484    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
19485    /// own top-of-function fast path (`review_conclusion`) correctly reads
19486    /// this shape as `None` rather than guessing `Blocked`, and the loop's
19487    /// own `for` range is empty once the round budget is spent, so
19488    /// `review_loop` must retry the check itself rather than silently doing
19489    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
19490    /// above exercises directly, but through `review_loop`'s own entry point
19491    /// this time, proving the wiring between the two rather than just the
19492    /// retry logic in isolation.
19493    #[tokio::test]
19494    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
19495        crate::run::pin_test_home();
19496        let home = crate::run::home();
19497
19498        let tmp = tempfile::tempdir().expect("tempdir");
19499        let repo = tmp.path().join("repo");
19500        std::fs::create_dir_all(&repo).unwrap();
19501        init_repo(&repo);
19502        let head = crate::git::rev_parse(&repo, "HEAD")
19503            .await
19504            .expect("rev-parse");
19505        let cache_dir = tmp.path().join("target");
19506
19507        let mut config = Config::default();
19508        config.verify.e2e = vec![format!(
19509            "CARGO_TARGET_DIR='{}' test -f README.md",
19510            cache_dir.display()
19511        )];
19512        config.graph.review_rounds = 1;
19513        config.graph.timeout_verify = Some(2);
19514        config.graph.timeout_resource_wait = 2;
19515
19516        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19517        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19518            .expect("no io error acquiring directly")
19519        {
19520            crate::cache::AcquireOutcome::Acquired(g) => g,
19521            crate::cache::AcquireOutcome::Busy(b) => {
19522                panic!("expected the direct acquire to win the lease first: {b:?}")
19523            }
19524        };
19525
19526        let mut state = RunState::new(
19527            repo.clone(),
19528            "main".to_owned(),
19529            head.clone(),
19530            "task".to_owned(),
19531            config,
19532        );
19533        state.candidates = vec![Candidate {
19534            index: 0,
19535            label: 'A',
19536            agent: "alpha".to_owned(),
19537            branch: "does-not-exist".to_owned(),
19538            worktree: repo.clone(),
19539            summary: String::new(),
19540            stat: String::new(),
19541            files: 0,
19542            commits: 0,
19543            empty: false,
19544            failed: None,
19545            verified_noop: None,
19546            duration_ms: 0,
19547            folded: false,
19548        }];
19549        state.tally = Some(Tally {
19550            first_choice: BTreeMap::from([('A', 1)]),
19551            borda: BTreeMap::new(),
19552            winner: 'A',
19553            rankings: 1,
19554            unanimous_initial: true,
19555            deliberated: false,
19556            changed_votes: 0,
19557            unanimous_final: true,
19558            tie_break: None,
19559            judges: 0,
19560            present: 0,
19561            quorum: 0,
19562            met_quorum: true,
19563            uncontested: Some("only candidate A produced a change".to_owned()),
19564        });
19565        // The exact shape a prior process's `stop_reviewing` would have left
19566        // on disk: the round budget's last round, a real attempt already
19567        // made and already resource-blocked.
19568        state.reviews = vec![ReviewRound {
19569            round: 1,
19570            head: head.clone(),
19571            verified_head: Some(head.clone()),
19572            verified_at: Some(jiff::Timestamp::now()),
19573            reviews: Vec::new(),
19574            e2e: vec![CommandOutcome {
19575                command: format!(
19576                    "CARGO_TARGET_DIR='{}' test -f README.md",
19577                    cache_dir.display()
19578                ),
19579                code: None,
19580                output_tail: "waiting for the shared build cache".to_owned(),
19581                duration_ms: 0,
19582                resource_blocked: true,
19583            }],
19584            fix: None,
19585            blocking: 1,
19586            answered: 1,
19587            expected: 1,
19588            clean: false,
19589            verify_retried: false,
19590            e2e_deferred: false,
19591            e2e_defer_reason: None,
19592            progressed: false,
19593            vote_split: false,
19594            reconsideration: Vec::new(),
19595            verdict: None,
19596        }];
19597
19598        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
19599        let mut runner = Runner {
19600            state,
19601            roles: ResolvedRoles {
19602                implementers: Vec::new(),
19603                judges: Vec::new(),
19604                reviewers: Vec::new(),
19605                fixer: None,
19606                conductor: conductor(),
19607                implementer_roster: Vec::new(),
19608                judge_roster: Vec::new(),
19609                reviewer_roster: Vec::new(),
19610            },
19611            sem: Arc::new(Semaphore::new(1)),
19612            pause: Pause::new(),
19613            interrupt: Pause::new(),
19614        };
19615
19616        // The lease is still held throughout, so this reentry's own retry is
19617        // also contended — proving `review_loop` actually tried again (not
19618        // that it happened to succeed) is what the timestamp comparison
19619        // below is for.
19620        runner.review_loop().await.expect("review_loop");
19621
19622        assert_eq!(
19623            runner.state.reviews.len(),
19624            1,
19625            "no new round was started on top of the unresolved one: {:?}",
19626            runner.state.reviews
19627        );
19628        let last = &runner.state.reviews[0];
19629        assert_eq!(
19630            last.e2e_status(),
19631            E2eStatus::ResourceBlocked,
19632            "still contended: {last:?}"
19633        );
19634        assert!(
19635            last.verified_at.expect("still known") > first_attempt_at,
19636            "review_loop must have actually retried the check, not left it exactly as found"
19637        );
19638        assert_ne!(
19639            runner.state.status,
19640            RunStatus::Blocked,
19641            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
19642            runner.state.status
19643        );
19644
19645        held.release();
19646    }
19647
19648    #[tokio::test]
19649    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
19650        crate::run::pin_test_home();
19651        for status in [RunStatus::Blocked, RunStatus::Gating] {
19652            let mut runner = runner_at(status);
19653            runner.state.config.graph.land = true;
19654            runner.state.gate_ran = true;
19655            runner.state.merge = Some(MergeOutcome {
19656                mode: MergeMode::Pr,
19657                ok: false,
19658                detail: "the fixer produced no commit".into(),
19659                empty: false,
19660            });
19661            let url = "https://example.invalid/o/r/pull/1";
19662            runner.state.pr = Some(crate::run::PrRecord {
19663                url: url.into(),
19664                number: 1,
19665                state: "open".into(),
19666                checks: "red".into(),
19667                round: 1,
19668                rounds: 1,
19669                red_at_merge: Vec::new(),
19670            });
19671            runner.state.save().unwrap();
19672            // The nonexistent repository makes observation fail before a forge
19673            // command can run. We still must reach land, rather than silently
19674            // return from the graph with its old gate and merge records.
19675            runner.execute_graph().await.unwrap();
19676            assert_eq!(runner.state.status, RunStatus::Blocked);
19677            assert!(runner.state.land_sync_pending);
19678            assert!(runner.state.events.iter().any(|event|
19679                event.node == "land" && event.message == format!("watching {url}")));
19680            assert_eq!(
19681                runner.state.merge.as_ref().unwrap().detail,
19682                "the fixer produced no commit"
19683            );
19684        }
19685    }
19686
19687    #[tokio::test]
19688    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
19689        crate::run::pin_test_home();
19690        let tmp = tempfile::tempdir().expect("tempdir");
19691        let repo = tmp.path().join("repo");
19692        std::fs::create_dir_all(&repo).unwrap();
19693        init_repo(&repo);
19694
19695        let mut config = Config::default();
19696        config.merge.mode = MergeMode::Pr;
19697        config.graph.land = true;
19698        config.graph.land_approval = false;
19699
19700        let mut state = RunState::new(
19701            repo.clone(),
19702            "main".to_owned(),
19703            "deadbeef".to_owned(),
19704            "task".to_owned(),
19705            config,
19706        );
19707        state.candidates = vec![Candidate {
19708            index: 0,
19709            label: 'A',
19710            agent: "alpha".to_owned(),
19711            branch: "does-not-exist".to_owned(),
19712            worktree: repo.clone(),
19713            summary: String::new(),
19714            stat: String::new(),
19715            files: 0,
19716            commits: 0,
19717            empty: false,
19718            failed: None,
19719            verified_noop: None,
19720            duration_ms: 0,
19721            folded: false,
19722        }];
19723        state.tally = Some(Tally {
19724            first_choice: BTreeMap::from([('A', 1)]),
19725            borda: BTreeMap::new(),
19726            winner: 'A',
19727            rankings: 1,
19728            unanimous_initial: true,
19729            deliberated: false,
19730            changed_votes: 0,
19731            unanimous_final: true,
19732            tie_break: None,
19733            judges: 0,
19734            present: 0,
19735            quorum: 0,
19736            met_quorum: true,
19737            uncontested: Some("only candidate A produced a change".to_owned()),
19738        });
19739        state.reviews = vec![ReviewRound {
19740            round: 1,
19741            head: "deadbeef".to_owned(),
19742            verified_head: None,
19743            verified_at: None,
19744            reviews: Vec::new(),
19745            e2e: Vec::new(),
19746            fix: None,
19747            blocking: 0,
19748            answered: 0,
19749            expected: 0,
19750            clean: true,
19751            verify_retried: false,
19752            e2e_deferred: false,
19753            e2e_defer_reason: None,
19754            progressed: false,
19755            vote_split: false,
19756            reconsideration: Vec::new(),
19757            verdict: None,
19758        }];
19759        state.gate = vec![CommandOutcome {
19760            command: "test".to_owned(),
19761            code: Some(0),
19762            output_tail: String::new(),
19763            duration_ms: 0,
19764            resource_blocked: false,
19765        }];
19766        state.gate_ran = true;
19767        // A first pass through `merge` already pushed and opened this pull
19768        // request; `status` is `Landing` because a previous call into `land`
19769        // parked or was interrupted before it reached a terminal outcome.
19770        state.status = RunStatus::Landing;
19771        state.merge = Some(MergeOutcome {
19772            mode: MergeMode::Pr,
19773            ok: true,
19774            detail: "https://example.invalid/x/y/pull/1".to_owned(),
19775            empty: false,
19776        });
19777
19778        // The Landing-resume shortcut calls `run_land` directly rather than
19779        // through `merge`, which is exactly the call site that used to skip
19780        // `settle_questions` - see the fixture below.
19781        ask_test_home();
19782        // A real resume loads a persisted run; driver ownership is checked
19783        // against that record before the Landing shortcut is entered.
19784        state.save().expect("save landing run");
19785        let store = ask::Questions::open();
19786        let q = ask_open_question(&store, &state.id);
19787
19788        let mut runner = Runner {
19789            state,
19790            roles: ResolvedRoles {
19791                implementers: Vec::new(),
19792                judges: Vec::new(),
19793                reviewers: Vec::new(),
19794                fixer: None,
19795                conductor: conductor(),
19796                implementer_roster: Vec::new(),
19797                judge_roster: Vec::new(),
19798                reviewer_roster: Vec::new(),
19799            },
19800            sem: Arc::new(Semaphore::new(1)),
19801            pause: Pause::new(),
19802            interrupt: Pause::new(),
19803        };
19804
19805        // `execute`, not `merge` directly: the Landing-resume shortcut lives
19806        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
19807        // exactly because `review_loop` would otherwise clobber the marker
19808        // first.
19809        runner.execute().await.expect("execute");
19810
19811        assert_eq!(
19812            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19813            Some("https://example.invalid/x/y/pull/1"),
19814            "reentry must not push again or open a second pull request over the \
19815             one `land` is already watching"
19816        );
19817        assert_ne!(
19818            runner.state.status,
19819            RunStatus::Landing,
19820            "land could not actually reach the fake pull request, so it must \
19821             have given up rather than left the run silently parked forever"
19822        );
19823        // `land` could not reach the fake pull request, so it gave up into
19824        // `Blocked` - still resumable, so the question must not have been
19825        // swept just because this branch now also calls `settle_questions`.
19826        assert_eq!(runner.state.status, RunStatus::Blocked);
19827        assert!(
19828            store.get(&q.id).unwrap().status.open(),
19829            "Blocked is still alive; settle_questions must have been a no-op here"
19830        );
19831    }
19832
19833    fn state_with_round(round: ReviewRound) -> RunState {
19834        let mut s = RunState::new(
19835            PathBuf::from("/repo"),
19836            "main".to_owned(),
19837            "abc1234".to_owned(),
19838            "add retries".to_owned(),
19839            Config::default(),
19840        );
19841        s.reviews = vec![round];
19842        s
19843    }
19844
19845    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
19846        crate::verdict::Finding {
19847            id: id.to_owned(),
19848            severity,
19849            file: None,
19850            line: None,
19851            title: title.to_owned(),
19852            detail: String::new(),
19853        }
19854    }
19855
19856    #[test]
19857    fn pr_body_names_open_findings_and_declined_ones() {
19858        let round = ReviewRound {
19859            round: 2,
19860            head: "deadbee".to_owned(),
19861            verified_head: None,
19862            verified_at: None,
19863            reviews: vec![ReviewRecord {
19864                attempts: 0,
19865                reviewer: 1,
19866                agent: "alpha".to_owned(),
19867                summary: String::new(),
19868                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
19869                vote: None,
19870                failed: None,
19871                duration_ms: 0,
19872            }],
19873            e2e: vec![CommandOutcome {
19874                command: "cargo test".to_owned(),
19875                code: Some(0),
19876                output_tail: String::new(),
19877                duration_ms: 0,
19878                resource_blocked: false,
19879            }],
19880            verify_retried: false,
19881            e2e_deferred: false,
19882            e2e_defer_reason: None,
19883            fix: Some(FixRecord {
19884                agent: "alpha".to_owned(),
19885                addressed: Vec::new(),
19886                rejected: vec![crate::verdict::Rejection {
19887                    id: "R1-1-1".to_owned(),
19888                    why: "not reachable from any caller".to_owned(),
19889                }],
19890                notes: String::new(),
19891                committed: true,
19892                failed: None,
19893                duration_ms: 0,
19894                continuation: None,
19895            }),
19896            blocking: 0,
19897            answered: 1,
19898            expected: 1,
19899            clean: false,
19900            progressed: true,
19901            vote_split: false,
19902            reconsideration: Vec::new(),
19903            verdict: None,
19904        };
19905        let state = state_with_round(round);
19906        let body = pr_message(&state, 'A').body;
19907
19908        assert!(body.contains("add retries"), "the task must still be there");
19909        assert!(body.contains("R2-1-1"), "{body}");
19910        assert!(!body.contains("unused import"), "{body}");
19911        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
19912        assert!(
19913            !body.contains("not reachable from any caller"),
19914            "the reason it was declined: {body}"
19915        );
19916    }
19917
19918    #[test]
19919    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
19920        let round = ReviewRound {
19921            round: 1,
19922            head: "deadbee".to_owned(),
19923            verified_head: None,
19924            verified_at: None,
19925            reviews: vec![ReviewRecord {
19926                attempts: 0,
19927                reviewer: 1,
19928                agent: "alpha".to_owned(),
19929                summary: String::new(),
19930                findings: Vec::new(),
19931                vote: None,
19932                failed: None,
19933                duration_ms: 0,
19934            }],
19935            e2e: Vec::new(),
19936            verify_retried: false,
19937            e2e_deferred: false,
19938            e2e_defer_reason: None,
19939            fix: None,
19940            blocking: 0,
19941            answered: 1,
19942            expected: 1,
19943            clean: true,
19944            progressed: false,
19945            vote_split: false,
19946            reconsideration: Vec::new(),
19947            verdict: None,
19948        };
19949        let state = state_with_round(round);
19950        let body = pr_message(&state, 'A').body;
19951        assert!(!body.contains("Open review findings"), "{body}");
19952        assert!(!body.contains("Declined"), "{body}");
19953    }
19954
19955    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
19956        let mut state = RunState::new(
19957            PathBuf::from("/repo"),
19958            "main".to_owned(),
19959            "abc1234".to_owned(),
19960            instruction.to_owned(),
19961            Config::default(),
19962        );
19963        state.candidates.push(Candidate {
19964            index: 0,
19965            label: 'A',
19966            agent: "alpha".to_owned(),
19967            branch: "magi/x/A".to_owned(),
19968            worktree: PathBuf::from("/wt"),
19969            summary: summary.to_owned(),
19970            stat: String::new(),
19971            files: 1,
19972            commits: 1,
19973            empty: false,
19974            failed: None,
19975            verified_noop: None,
19976            folded: false,
19977            duration_ms: 0,
19978        });
19979        state
19980    }
19981
19982    fn review_state(subjects: &[&str]) -> RunState {
19983        let mut state = state_with_summary(
19984            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
19985            "",
19986        );
19987        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19988        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
19989        state
19990    }
19991
19992    /// A branch rebased onto a main that moved past the recorded
19993    /// `base_commit` is titled from its own first commit, never main's.
19994    #[tokio::test]
19995    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
19996        ask_test_home();
19997        let tmp = tempfile::tempdir().unwrap();
19998        let repo = tmp.path().join("repo");
19999        std::fs::create_dir_all(&repo).unwrap();
20000        init_repo(&repo);
20001        let origin = tmp.path().join("origin.git");
20002        let g = |dir: &Path, args: &[&str]| {
20003            let out = std::process::Command::new("git")
20004                .args(args)
20005                .current_dir(dir)
20006                .quiet()
20007                .output()
20008                .expect("spawn git");
20009            assert!(
20010                out.status.success(),
20011                "git {args:?}: {}",
20012                String::from_utf8_lossy(&out.stderr)
20013            );
20014        };
20015        g(
20016            tmp.path(),
20017            &[
20018                "clone",
20019                "--bare",
20020                "-q",
20021                repo.to_str().unwrap(),
20022                origin.to_str().unwrap(),
20023            ],
20024        );
20025        g(
20026            &repo,
20027            &["remote", "add", "origin", origin.to_str().unwrap()],
20028        );
20029        let c1 = git::rev_parse(&repo, "main").await.unwrap();
20030
20031        // Main moves on; the branch is built on top of the new main.
20032        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
20033        g(&repo, &["add", "-A"]);
20034        g(
20035            &repo,
20036            &["commit", "-q", "-m", "chore(deps): update a crate"],
20037        );
20038        g(&repo, &["push", "-q", "origin", "main"]);
20039        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
20040        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
20041        g(&repo, &["add", "-A"]);
20042        g(
20043            &repo,
20044            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
20045        );
20046        g(&repo, &["checkout", "-q", "main"]);
20047
20048        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
20049        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
20050        // Without a readable tracking ref the recorded base's merge base is used.
20051        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
20052        assert_eq!(fallback, c1);
20053
20054        let mut state = review_state(&[
20055            "chore(deps): update a crate",
20056            "fix(daemon): apply a chosen action",
20057        ]);
20058        state.repo = repo.clone();
20059        state.base_branch = "main".to_owned();
20060        state.base_commit = c1;
20061        refresh_reviewed_commits(&mut state, "feat/own").await;
20062        assert_eq!(
20063            state.reviewed_commits,
20064            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
20065        );
20066        assert_eq!(
20067            review_title(&state).as_deref(),
20068            Some("fix(daemon): apply a chosen action")
20069        );
20070        assert_eq!(
20071            leaked_subjects(&state, "feat/own").await,
20072            Some(vec!["chore(deps): update a crate".to_owned()])
20073        );
20074        // A stale tracking ref is still used when the fetch fails, but the
20075        // leak list is withheld.
20076        state.config.merge.remote = "nowhere".to_owned();
20077        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
20078    }
20079
20080    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
20081    /// with `origin/main` one commit behind it, and a summarizing agent that
20082    /// prints whatever `reply` holds.
20083    struct SummaryFixture {
20084        _tmp: tempfile::TempDir,
20085        repo: PathBuf,
20086        reply: PathBuf,
20087        runner: Runner,
20088    }
20089
20090    fn summary_fixture() -> SummaryFixture {
20091        ask_test_home();
20092        let tmp = tempfile::tempdir().unwrap();
20093        let repo = tmp.path().join("repo");
20094        std::fs::create_dir_all(&repo).unwrap();
20095        init_repo(&repo);
20096        let origin = tmp.path().join("origin.git");
20097        let g = |args: &[&str]| {
20098            let out = std::process::Command::new("git")
20099                .args(args)
20100                .current_dir(&repo)
20101                .quiet()
20102                .output()
20103                .expect("spawn git");
20104            assert!(
20105                out.status.success(),
20106                "git {args:?}: {}",
20107                String::from_utf8_lossy(&out.stderr)
20108            );
20109        };
20110        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
20111        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
20112        g(&["push", "-q", "origin", "main"]);
20113        g(&["checkout", "-q", "-b", "feat/own"]);
20114        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
20115        g(&["add", "-A"]);
20116        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
20117        let reply = tmp.path().join("reply.txt");
20118        std::fs::write(
20119            &reply,
20120            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
20121        )
20122        .unwrap();
20123        let mut runner = runner_at(RunStatus::Gating);
20124        let mut state = review_state(&["feat(own): add the own file"]);
20125        state.repo = repo.clone();
20126        state.base_branch = "main".to_owned();
20127        state.candidates[0].branch = "feat/own".to_owned();
20128        state.candidates[0].worktree = repo.clone();
20129        let mut author = spec("alpha");
20130        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
20131        author
20132            .env
20133            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
20134        state.config.agents = vec![author];
20135        runner.state = state;
20136        SummaryFixture {
20137            _tmp: tmp,
20138            repo,
20139            reply,
20140            runner,
20141        }
20142    }
20143
20144    async fn posted(f: &mut SummaryFixture) -> PrMessage {
20145        let winner = f.runner.state.candidates[0].clone();
20146        f.runner
20147            .guarded_pr_message(&winner, None, true)
20148            .await
20149            .unwrap()
20150            .expect("an English description never parks")
20151    }
20152
20153    fn clean_round_with_minor_findings() -> ReviewRound {
20154        let mut round = passing_round(2, None);
20155        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
20156        round.reviews = vec![ReviewRecord {
20157            attempts: 0,
20158            reviewer: 1,
20159            agent: "alpha".into(),
20160            summary: "内部レビュー".into(),
20161            findings: vec![
20162                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
20163                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
20164            ],
20165            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
20166            failed: None,
20167            duration_ms: 0,
20168        }];
20169        round
20170    }
20171
20172    #[tokio::test]
20173    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
20174        for review_only in [false, true] {
20175            let mut f = summary_fixture();
20176            if !review_only {
20177                f.runner.state.candidates[0].agent = "alpha".into();
20178            }
20179            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
20180            assert!(f.runner.state.reviews[0].clean);
20181            assert!(f.runner.state.reviews[0].fix.is_none());
20182            assert!(f.runner.state.open_findings().is_empty());
20183            assert_eq!(
20184                crate::pr_outcome::required_finding_ids(&f.runner.state),
20185                ["R2-1-1", "R2-1-2"]
20186            );
20187            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20188            let evidence = crate::pr_outcome::summary_evidence(
20189                &f.runner.state,
20190                crate::pr_outcome::FinalHead::At(&head),
20191            );
20192            assert!(
20193                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
20194            );
20195            // The ordinary reply omits both IDs: no cached description is adopted.
20196            let omitted = posted(&mut f).await;
20197            assert!(omitted.body.contains("No description"));
20198            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
20199            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
20200            assert!(f.runner.state.change_summary.is_none());
20201            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
20202            let described = posted(&mut f).await;
20203            assert!(described.body.contains("improve documentation for callers"));
20204            assert!(described.body.contains("remaining edge case"));
20205            assert!(described.body.contains("## Open review findings"));
20206            assert!(!described.body.contains("未修正"));
20207        }
20208    }
20209
20210    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
20211        let mut f = summary_fixture();
20212        if !review_only {
20213            f.runner.state.candidates[0].agent = "alpha".into();
20214        }
20215        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20216        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
20217        fixed
20218            .fix
20219            .as_mut()
20220            .unwrap()
20221            .rejected
20222            .push(crate::verdict::Rejection {
20223                id: "R1-1-1".into(),
20224                why: "呼び出されないコードのため辞退".into(),
20225            });
20226        let mut clean = clean_round_with_minor_findings();
20227        clean.verified_head = Some(head.clone());
20228        f.runner.state.reviews = vec![fixed, clean];
20229        ok_gate(&mut f.runner.state, Some(&head));
20230        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
20231        f.runner.state.config.agents.push(scripted("beta", reply));
20232        f.runner.state.config.roles.language_judge =
20233            Some(crate::config::AgentChoice::One("beta".into()));
20234        f
20235    }
20236
20237    #[tokio::test]
20238    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
20239        for review_only in [false, true] {
20240            let mut f = rewrite_fixture_with_findings(review_only,
20241                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
20242            let m = posted(&mut f).await;
20243            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20244            let expected =
20245                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
20246            assert!(m.body.contains(&expected));
20247            assert!(m.body.contains("## Review rounds"));
20248            assert!(
20249                m.body
20250                    .contains("Gate (`verify.gate`): passed on the final commit")
20251            );
20252            assert!(
20253                m.body
20254                    .contains("Tests (`verify.e2e`): passed on the final commit")
20255            );
20256            let c = f.runner.state.change_summary.as_ref().unwrap();
20257            assert!(
20258                m.body
20259                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
20260            );
20261            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
20262            assert!(crate::pr_outcome::covers_required_findings(
20263                &f.runner.state,
20264                &m.body
20265            ));
20266            assert!(m.body.contains("documentation remains incomplete"));
20267            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
20268            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
20269            assert!(f.runner.state.github_text.is_none());
20270        }
20271    }
20272
20273    #[tokio::test]
20274    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
20275        for review_only in [false, true] {
20276            for reply in [
20277                "TITLE: fix: public description\nA short English description.",
20278                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
20279            ] {
20280                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
20281                let winner = f.runner.state.candidates[0].clone();
20282                let posted = f
20283                    .runner
20284                    .guarded_pr_message(&winner, None, true)
20285                    .await
20286                    .unwrap();
20287                assert!(posted.is_none());
20288                assert!(f.runner.state.parked);
20289                let withheld = f.runner.state.github_text.as_ref().unwrap();
20290                assert!(
20291                    withheld.rewrite.is_none(),
20292                    "an incomplete rewrite must not be selectable"
20293                );
20294                let q = ask::Questions::open()
20295                    .get(withheld.question.as_ref().unwrap())
20296                    .unwrap();
20297                assert!(
20298                    !q.choices
20299                        .contains(&crate::github_text::USE_REWRITE.to_owned())
20300                );
20301                assert!(
20302                    q.detail.contains("R1-1-1")
20303                        && q.detail.contains("R2-1-1")
20304                        && q.detail.contains("R2-1-2")
20305                );
20306            }
20307        }
20308    }
20309
20310    #[tokio::test]
20311    async fn japanese_reports_stay_local_in_both_public_run_modes() {
20312        for review_only in [false, true] {
20313            let mut f = summary_fixture();
20314            if !review_only {
20315                f.runner.state.candidates[0].agent = "alpha".into();
20316            }
20317            f.runner.state.instruction = "内部タスクの説明".into();
20318            f.runner.state.candidates[0].summary = "内部の候補報告".into();
20319            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20320            f.runner.state.reviews = vec![
20321                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
20322                round_with_notes(2, Some("二回目の内部修正報告です。")),
20323                passing_round(3, Some(&head)),
20324            ];
20325            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
20326            ok_gate(&mut f.runner.state, Some(&head));
20327            let captured = f.repo.join("public-prompt.txt");
20328            let author = &mut f.runner.state.config.agents[0];
20329            author.command = vec![
20330                "sh".into(),
20331                "-c".into(),
20332                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
20333            ];
20334            author.env.insert(
20335                "PROMPT_FILE".into(),
20336                captured.to_string_lossy().into_owned(),
20337            );
20338            let m = posted(&mut f).await;
20339            assert!(m.body.contains("Adds `own.txt`"));
20340            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
20341            assert!(m.body.contains("R1-1-1"));
20342            assert!(
20343                m.body
20344                    .contains("Gate (`verify.gate`): passed on the final commit")
20345            );
20346            assert!(
20347                m.body
20348                    .contains("Tests (`verify.e2e`): passed on the final commit")
20349            );
20350            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
20351            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
20352            let prompt = std::fs::read_to_string(captured).unwrap();
20353            assert!(prompt.contains(&format!(
20354                "git diff {} {head}",
20355                f.runner.state.change_summary.as_ref().unwrap().base
20356            )));
20357            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
20358            assert_eq!(
20359                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
20360                "二回目の内部修正報告です。"
20361            );
20362        }
20363    }
20364
20365    #[tokio::test]
20366    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
20367        for review_only in [false, true] {
20368            let mut f = summary_fixture();
20369            if !review_only {
20370                f.runner.state.candidates[0].agent = "alpha".into();
20371            }
20372            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
20373            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
20374            std::fs::write(&f.reply, "no usable description").unwrap();
20375            let failed = posted(&mut f).await;
20376            assert!(
20377                failed
20378                    .body
20379                    .contains("No description of what this change does could be generated")
20380            );
20381            assert!(failed.body.is_ascii());
20382            assert!(f.runner.state.github_text.is_none());
20383            assert!(f.runner.state.change_summary.is_none());
20384            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20385            let evidence = crate::pr_outcome::summary_evidence(
20386                &f.runner.state,
20387                crate::pr_outcome::FinalHead::At(&head),
20388            );
20389            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
20390            assert!(
20391                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
20392                    .exists()
20393            );
20394            f.runner.state =
20395                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
20396            std::fs::write(
20397                &f.reply,
20398                "TITLE: feat(own): add the own file\nAdds the committed own file.",
20399            )
20400            .unwrap();
20401            let retried = posted(&mut f).await;
20402            assert!(retried.body.contains("Adds the committed own file."));
20403            assert_eq!(
20404                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
20405                "日本語の修正報告"
20406            );
20407        }
20408    }
20409
20410    #[tokio::test]
20411    async fn same_head_verification_and_findings_invalidate_public_prose() {
20412        let mut f = summary_fixture();
20413        posted(&mut f).await;
20414        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20415        let original = f
20416            .runner
20417            .state
20418            .change_summary
20419            .as_ref()
20420            .unwrap()
20421            .evidence
20422            .clone();
20423        ok_gate(&mut f.runner.state, Some(&head));
20424        std::fs::write(
20425            &f.reply,
20426            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
20427        )
20428        .unwrap();
20429        let verified = posted(&mut f).await;
20430        assert!(verified.body.contains("now verified"));
20431        assert_ne!(
20432            f.runner.state.change_summary.as_ref().unwrap().evidence,
20433            original
20434        );
20435        let mut round = round_with_notes(1, Some("内部報告"));
20436        round.reviews.push(ReviewRecord {
20437            attempts: 0,
20438            reviewer: 1,
20439            agent: "alpha".into(),
20440            summary: "内部レビュー".into(),
20441            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
20442            vote: None,
20443            failed: None,
20444            duration_ms: 0,
20445        });
20446        round
20447            .fix
20448            .as_mut()
20449            .unwrap()
20450            .rejected
20451            .push(crate::verdict::Rejection {
20452                id: "R1-1-1".into(),
20453                why: "呼び出し元から到達できない".into(),
20454            });
20455        f.runner.state.reviews = vec![round];
20456        // A description omitting the required ID is unusable, even with a title.
20457        let omitted = posted(&mut f).await;
20458        assert!(omitted.body.contains("No description"));
20459        assert!(omitted.body.contains("R1-1-1"));
20460        assert!(!omitted.body.contains("到達"));
20461        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
20462        let described = posted(&mut f).await;
20463        assert!(described.body.contains("unreachable from callers"));
20464        assert!(described.body.contains("R1-1-1"));
20465        assert!(!described.body.contains("到達"));
20466        let evidence = f
20467            .runner
20468            .state
20469            .change_summary
20470            .as_ref()
20471            .unwrap()
20472            .evidence
20473            .clone();
20474        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
20475        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
20476        assert!(posted(&mut f).await.body.contains("rationale updated"));
20477        assert_ne!(
20478            f.runner.state.change_summary.as_ref().unwrap().evidence,
20479            evidence
20480        );
20481        // Old serialized summaries have no evidence fingerprint and must regenerate.
20482        f.runner
20483            .state
20484            .change_summary
20485            .as_mut()
20486            .unwrap()
20487            .evidence
20488            .clear();
20489        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
20490        assert!(posted(&mut f).await.body.contains("current specification"));
20491    }
20492
20493    #[tokio::test]
20494    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
20495        let mut f = summary_fixture();
20496        let m = posted(&mut f).await;
20497        assert_eq!(m.title, "feat(own): add the own file");
20498        assert!(
20499            m.body
20500                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
20501            "{}",
20502            m.body
20503        );
20504        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
20505        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
20506        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
20507        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20508        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
20509        let kept = f.runner.state.change_summary.as_ref().unwrap();
20510        assert_eq!(
20511            (kept.head.as_str(), kept.base.as_str()),
20512            (head.as_str(), base.as_str())
20513        );
20514        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
20515    }
20516
20517    #[tokio::test]
20518    async fn review_only_summary_receives_the_configured_github_language() {
20519        let mut f = summary_fixture();
20520        f.runner.state.config.graph.github_language = "ja".into();
20521        let captured = f.repo.join("summary-prompt.txt");
20522        let author = &mut f.runner.state.config.agents[0];
20523        author.command = vec![
20524            "sh".into(),
20525            "-c".into(),
20526            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
20527        ];
20528        author.env.insert(
20529            "PROMPT_FILE".into(),
20530            captured.to_string_lossy().into_owned(),
20531        );
20532        std::fs::write(
20533            &f.reply,
20534            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
20535        )
20536        .unwrap();
20537        let m = posted(&mut f).await;
20538        let prompt = std::fs::read_to_string(captured).unwrap();
20539        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
20540        assert!(!prompt.contains("in English"));
20541        assert_eq!(m.title, "feat(own): ファイルを追加");
20542        assert!(m.body.contains("own.txt を追加します。"));
20543    }
20544
20545    #[tokio::test]
20546    async fn review_fixes_come_after_the_whole_change_summary() {
20547        let mut f = summary_fixture();
20548        f.runner.state.reviews = vec![
20549            round_with_notes(1, Some("handled the PATH case")),
20550            round_with_notes(2, None),
20551        ];
20552        let m = posted(&mut f).await;
20553        let summary = m.body.find("## Summary").expect("summary");
20554        let fixes = m.body.find("## Review rounds").expect("review rounds");
20555        assert!(summary < fixes, "{}", m.body);
20556        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
20557        assert!(m.body.contains("## Final verification"), "{}", m.body);
20558    }
20559
20560    #[tokio::test]
20561    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
20562        let mut f = summary_fixture();
20563        let first = posted(&mut f).await;
20564        let first_head = f.runner.state.change_summary.clone().unwrap().head;
20565        // The same head reuses the record: a different reply is not read.
20566        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
20567        let again = posted(&mut f).await;
20568        assert_eq!((again.title, again.body), (first.title, first.body));
20569
20570        let g = |args: &[&str]| {
20571            let out = std::process::Command::new("git")
20572                .args(args)
20573                .current_dir(&f.repo)
20574                .quiet()
20575                .output()
20576                .expect("spawn git");
20577            assert!(out.status.success(), "git {args:?}");
20578        };
20579        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
20580        g(&["add", "-A"]);
20581        g(&["commit", "-q", "-m", "fix(own): handle more"]);
20582        let second = posted(&mut f).await;
20583        let kept = f.runner.state.change_summary.clone().unwrap();
20584        assert_ne!(kept.head, first_head);
20585        assert!(second.body.contains("Not read yet."), "{}", second.body);
20586        assert!(!second.body.contains("so the first summary is read"));
20587        assert_eq!(second.title, "feat(own): other");
20588    }
20589
20590    #[tokio::test]
20591    async fn an_unavailable_summary_is_stated_and_never_invented() {
20592        let mut f = summary_fixture();
20593        std::fs::write(&f.reply, "no title line here").unwrap();
20594        let m = posted(&mut f).await;
20595        assert!(
20596            m.body
20597                .contains("No description of what this change does could be generated"),
20598            "{}",
20599            m.body
20600        );
20601        assert!(m.body.contains("internal reports remain in local records"));
20602        assert!(f.runner.state.change_summary.is_none());
20603        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
20604        // A preview asks nobody and claims nothing.
20605        let winner = f.runner.state.candidates[0].clone();
20606        let preview = f
20607            .runner
20608            .guarded_pr_message(&winner, None, false)
20609            .await
20610            .unwrap()
20611            .unwrap();
20612        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
20613    }
20614
20615    #[test]
20616    fn pr_message_review_single_commit_uses_its_subject() {
20617        let state = review_state(&["feat(nats): per-role user"]);
20618        let m = pr_message(&state, 'A');
20619        assert_eq!(m.title, "feat(nats): per-role user");
20620        assert!(!m.body.contains("Review the work already"), "{}", m.body);
20621        assert!(m.body.contains("## Commits under review"), "{}", m.body);
20622    }
20623
20624    #[test]
20625    fn pr_message_review_multi_commit_takes_the_oldest() {
20626        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
20627        let m = pr_message(&state, 'A');
20628        assert_eq!(m.title, "feat: the change");
20629        for s in ["feat: the change", "fix: typo", "fix: again"] {
20630            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
20631        }
20632    }
20633
20634    #[test]
20635    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
20636        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
20637            let mut state = review_state(&[first, "fix: later fixup"]);
20638            state.candidates[0].branch = "機能/ログイン".to_owned();
20639            let m = pr_message(&state, 'A');
20640            assert!(
20641                m.title.starts_with("chore: land candidate A of run"),
20642                "{}",
20643                m.title
20644            );
20645        }
20646    }
20647
20648    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
20649        BranchFacts {
20650            commits: commits
20651                .iter()
20652                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
20653                .collect(),
20654            stat: stat.to_owned(),
20655        }
20656    }
20657
20658    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
20659        let mut r = review_round(true, 0, 1, 1, true, true);
20660        r.round = round;
20661        r.fix = notes.map(|n| FixRecord {
20662            agent: "fixer".to_owned(),
20663            addressed: Vec::new(),
20664            rejected: Vec::new(),
20665            notes: n.to_owned(),
20666            committed: true,
20667            failed: None,
20668            duration_ms: 0,
20669            continuation: None,
20670        });
20671        r
20672    }
20673
20674    #[test]
20675    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
20676        let state = review_state(&["ignored"]);
20677        let f = facts(
20678            &[
20679                (
20680                    "fix(login): resolve PATH on macOS",
20681                    "Login shells skip rc files.",
20682                ),
20683                ("fix: address review", ""),
20684            ],
20685            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
20686        );
20687        let m = pr_message_with(&state, 'A', Some(&f));
20688        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
20689        assert!(
20690            m.body
20691                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
20692            "{}",
20693            m.body
20694        );
20695        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
20696        assert!(m.body.contains("## Diff stat"), "{}", m.body);
20697        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
20698        for banned in [
20699            "Review the work already",
20700            "no task statement",
20701            "Original task",
20702        ] {
20703            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
20704        }
20705        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
20706    }
20707
20708    #[test]
20709    fn pr_message_review_truncates_a_huge_first_commit_body() {
20710        let state = review_state(&["ignored"]);
20711        let f = facts(
20712            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
20713            "s",
20714        );
20715        let m = pr_message_with(&state, 'A', Some(&f));
20716        assert!(m.body.len() < 30_000, "{}", m.body.len());
20717        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
20718        assert!(
20719            m.body.contains("- ... 1 more commit(s)"),
20720            "{}",
20721            m.body.len()
20722        );
20723        assert!(m.body.ends_with("magi:candidate-a\n"));
20724    }
20725
20726    #[test]
20727    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
20728        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
20729        assert_eq!(m.title, "feat: x");
20730        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
20731        assert!(!m.body.contains("Diff stat"), "{}", m.body);
20732        assert!(!m.body.contains("no task statement"), "{}", m.body);
20733    }
20734
20735    #[test]
20736    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
20737        let mut state = review_state(&["日本語の件名"]);
20738        state.candidates[0].branch = "fix/macos-login-path".to_owned();
20739        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
20740        state.candidates[0].branch = "機能/ログイン".to_owned();
20741        assert!(
20742            pr_message(&state, 'A')
20743                .title
20744                .starts_with("chore: land candidate A")
20745        );
20746    }
20747
20748    #[test]
20749    fn pr_message_review_fixes_survive_a_clean_final_round() {
20750        let mut state = review_state(&["feat: x"]);
20751        state.reviews = vec![
20752            round_with_notes(1, Some("handled the PATH case")),
20753            round_with_notes(2, None),
20754        ];
20755        let body = pr_message(&state, 'A').body;
20756        assert!(
20757            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
20758            "{body}"
20759        );
20760        assert!(body.contains("### Round 1"), "{body}");
20761
20762        state.reviews = vec![
20763            round_with_notes(1, Some("first fix")),
20764            round_with_notes(2, Some("")),
20765            round_with_notes(3, Some("second fix")),
20766            round_with_notes(4, None),
20767        ];
20768        let body = pr_message(&state, 'A').body;
20769        assert!(
20770            body.contains("### Round 1") && !body.contains("first fix"),
20771            "{body}"
20772        );
20773        assert!(
20774            body.contains("### Round 3") && !body.contains("second fix"),
20775            "{body}"
20776        );
20777        assert!(body.contains("### Round 2"), "{body}");
20778    }
20779
20780    const FINAL: &str = "abcdef1234567890";
20781
20782    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
20783        let mut r = round_with_notes(round, None);
20784        r.verified_head = verified.map(str::to_owned);
20785        r
20786    }
20787
20788    fn ok_gate(state: &mut RunState, head: Option<&str>) {
20789        state.gate = vec![CommandOutcome {
20790            command: "gate".to_owned(),
20791            code: Some(0),
20792            output_tail: String::new(),
20793            duration_ms: 0,
20794            resource_blocked: false,
20795        }];
20796        state.gate_ran = true;
20797        state.gate_head = head.map(str::to_owned);
20798    }
20799
20800    fn body_at(state: &RunState) -> String {
20801        pr_message_raw(
20802            state,
20803            'A',
20804            None,
20805            crate::pr_outcome::FinalHead::At(FINAL),
20806            ChangeNote::NotAsked,
20807        )
20808        .body
20809    }
20810
20811    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
20812
20813    #[test]
20814    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
20815        let mut state = review_state(&["feat: x"]);
20816        state.reviews = vec![
20817            round_with_notes(1, Some(SANDBOXED)),
20818            passing_round(2, Some(FINAL)),
20819        ];
20820        ok_gate(&mut state, Some(FINAL));
20821        let body = body_at(&state);
20822        assert!(!body.contains("Handled the PATH case."), "{body}");
20823        assert!(!body.contains("could not commit"), "{body}");
20824        assert!(!body.contains("could not run"), "{body}");
20825        assert!(
20826            body.contains("The fix is committed on the branch"),
20827            "{body}"
20828        );
20829        assert!(!body.contains("sandbox"), "{body}");
20830        assert!(
20831            body.contains("Tests (`verify.e2e`): passed on the final commit"),
20832            "{body}"
20833        );
20834        assert!(
20835            body.contains("Gate (`verify.gate`): passed on the final commit"),
20836            "{body}"
20837        );
20838        // The raw report is untouched in the record.
20839        assert!(
20840            state.reviews[0]
20841                .fix
20842                .as_ref()
20843                .unwrap()
20844                .notes
20845                .contains("could not commit")
20846        );
20847    }
20848
20849    #[test]
20850    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
20851        let mut state = review_state(&["feat: x"]);
20852        let mut r1 = round_with_notes(1, Some(SANDBOXED));
20853        r1.fix.as_mut().unwrap().committed = false;
20854        let mut r2 = passing_round(2, Some(FINAL));
20855        r2.e2e[0].code = Some(1);
20856        state.reviews = vec![r1, r2];
20857        let body = body_at(&state);
20858        assert!(!body.contains("could not commit"), "{body}");
20859        assert!(!body.contains("could not run"), "{body}");
20860        assert!(body.contains("This round produced no commit"), "{body}");
20861        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
20862        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
20863        assert!(!body.contains("passed"), "{body}");
20864    }
20865
20866    #[test]
20867    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
20868        let mut state = review_state(&["feat: x"]);
20869        state.reviews = vec![
20870            round_with_notes(1, Some(SANDBOXED)),
20871            passing_round(2, Some("0123456789abcdef")),
20872        ];
20873        ok_gate(&mut state, Some("0123456789abcdef"));
20874        let body = body_at(&state);
20875        assert!(!body.contains("HTTP tests could not run"), "{body}");
20876        // The commit claim is still settled by the structured record.
20877        assert!(!body.contains("could not commit"), "{body}");
20878        assert!(
20879            body.contains("passed on an earlier commit (`0123456`)"),
20880            "{body}"
20881        );
20882        assert!(!body.contains("passed on the final commit"), "{body}");
20883
20884        // A gate recorded before `gate_head` existed is "not recorded".
20885        ok_gate(&mut state, None);
20886        let body = body_at(&state);
20887        assert!(
20888            body.contains("passed on a commit that was not recorded"),
20889            "{body}"
20890        );
20891    }
20892
20893    #[test]
20894    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
20895        let state = state_with_summary(
20896            "add retries to the client",
20897            "TITLE: feat: retries\n\nDid it.",
20898        );
20899        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
20900        assert_eq!(m.title, "feat: retries");
20901        assert!(
20902            m.body.contains("<summary>Original task</summary>"),
20903            "{}",
20904            m.body
20905        );
20906        assert!(!m.body.contains("Commits under review"), "{}", m.body);
20907        assert!(
20908            m.body
20909                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
20910            "{}",
20911            m.body
20912        );
20913    }
20914
20915    #[test]
20916    fn pr_message_review_bounds_a_long_english_subject() {
20917        let long = format!("feat: {}", "word ".repeat(100));
20918        let m = pr_message(&review_state(&[&long]), 'A');
20919        assert!(m.title.starts_with("feat: word"), "{}", m.title);
20920        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20921    }
20922
20923    #[test]
20924    fn pr_message_implementation_run_is_unchanged_by_review_support() {
20925        let state = state_with_summary("add retries\n\ndetails", "- did some things");
20926        let m = pr_message(&state, 'A');
20927        assert_eq!(m.title, "add retries");
20928        assert!(m.body.contains("<summary>Original task</summary>"));
20929        assert!(!m.body.contains("Commits under review"));
20930        assert_eq!(landing_subject_source(&state), state.instruction);
20931    }
20932
20933    #[test]
20934    fn review_run_squash_subject_is_the_change_not_the_prompt() {
20935        let state = review_state(&["feat: the change", "fix: typo"]);
20936        let source = landing_subject_source(&state);
20937        assert_eq!(land::merge_subject("", &source), "feat: the change");
20938        assert_eq!(
20939            land::merge_subject("magi: candidate A (uncommitted work)", &source),
20940            "feat: the change"
20941        );
20942        // An operator's rename still wins.
20943        assert_eq!(
20944            land::merge_subject("feat: renamed by hand", &source),
20945            "feat: renamed by hand"
20946        );
20947        let blank = review_state(&["日本語"]);
20948        assert!(
20949            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
20950        );
20951    }
20952
20953    #[test]
20954    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
20955        let state = review_state(&["feat: the change"]);
20956        let source = landing_subject_source(&state);
20957        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
20958        assert_eq!(
20959            land::merge_subject(landing_title(&state, old), &source),
20960            "feat: the change"
20961        );
20962        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
20963        let task = state_with_summary("add retries", "");
20964        assert_eq!(landing_title(&task, old), old);
20965    }
20966
20967    #[test]
20968    fn pr_message_describes_the_change_not_the_task() {
20969        let state = state_with_summary(
20970            "今回やってほしいこと: results projector を直す",
20971            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
20972        );
20973        let m = pr_message(&state, 'A');
20974        assert_eq!(m.title, "fix(web): batch the runs list reads");
20975        assert!(
20976            m.body.starts_with("## Summary\n\n- reads run.json once"),
20977            "{}",
20978            m.body
20979        );
20980        assert!(!m.body.contains("TITLE:"), "{}", m.body);
20981        let task_at = m.body.find("今回やってほしいこと").unwrap();
20982        let details_at = m.body.find("<details>").unwrap();
20983        assert!(
20984            details_at < task_at,
20985            "the task lives inside <details>: {}",
20986            m.body
20987        );
20988        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
20989        assert!(m.body.contains("magi:candidate-a"));
20990    }
20991
20992    #[test]
20993    fn pr_message_falls_back_to_the_task_without_a_title_line() {
20994        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
20995        let m = pr_message(&state, 'A');
20996        assert_eq!(m.title, "add retries");
20997        assert!(
20998            m.body.contains("## Summary\n\n- did some things"),
20999            "{}",
21000            m.body
21001        );
21002
21003        let none = RunState::new(
21004            PathBuf::from("/repo"),
21005            "main".to_owned(),
21006            "abc1234".to_owned(),
21007            "add retries".to_owned(),
21008            Config::default(),
21009        );
21010        let m = pr_message(&none, 'A');
21011        assert_eq!(m.title, "add retries");
21012        assert!(!m.body.contains("## Summary"), "{}", m.body);
21013    }
21014
21015    #[test]
21016    fn pr_message_refuses_the_candidate_commit_subject() {
21017        for bad in [
21018            "TITLE: magi: candidate A (uncommitted work)",
21019            "TITLE: chore: stuff (uncommitted work)",
21020            "TITLE:   ",
21021        ] {
21022            let state = state_with_summary("add retries", bad);
21023            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
21024        }
21025    }
21026
21027    #[test]
21028    fn pr_message_bounds_a_very_long_task_and_title() {
21029        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
21030        let state = state_with_summary(&long, "- nothing");
21031        let m = pr_message(&state, 'A');
21032        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
21033        assert!(!m.title.contains('\n'));
21034
21035        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
21036        let m = pr_message(&state, 'A');
21037        assert!(m.title.starts_with("feat: "));
21038        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
21039        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
21040    }
21041
21042    fn long_title_of(instruction: &str) -> String {
21043        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
21044    }
21045
21046    #[test]
21047    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
21048        let first = "Make the landing path keep a readable title for long tasks";
21049        let line = format!(
21050            "{first}. {}",
21051            "And then keep going with more words ".repeat(20)
21052        );
21053        let t = long_title_of(&line);
21054        assert_eq!(t, first);
21055        assert!(!t.starts_with("chore: land"));
21056    }
21057
21058    #[test]
21059    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
21060        let line = "word ".repeat(200);
21061        let t = long_title_of(&line);
21062        assert!(t.ends_with("word..."), "{t}");
21063        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
21064    }
21065
21066    #[test]
21067    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
21068        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
21069            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
21070        }
21071    }
21072
21073    #[test]
21074    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
21075        let mut state = state_with_summary("再試行を追加する", "- nothing");
21076        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
21077        state.config.graph.github_language = "ja".into();
21078        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
21079    }
21080
21081    #[test]
21082    fn pr_message_title_limit_is_exact() {
21083        let at = "a".repeat(PR_TITLE_MAX);
21084        assert_eq!(long_title_of(&at), at);
21085        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
21086        assert!(over.ends_with("..."), "{over}");
21087        assert_eq!(over.chars().count(), PR_TITLE_MAX);
21088    }
21089
21090    #[test]
21091    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
21092        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
21093        let t = long_title_of(&line);
21094        assert!(t.ends_with("..."), "{t}");
21095        assert!(t.is_ascii(), "{t}");
21096    }
21097
21098    #[test]
21099    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
21100        let line = format!(
21101            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
21102            "plus more filler words ".repeat(20)
21103        );
21104        let t = long_title_of(&line);
21105        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
21106    }
21107
21108    #[test]
21109    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
21110        let line = format!("\n\n# {}", "title words ".repeat(40));
21111        let state = state_with_summary(&line, "- nothing");
21112        let m = pr_message(&state, 'A');
21113        assert!(m.title.starts_with("title words"), "{}", m.title);
21114        assert_eq!(
21115            land::merge_subject(&m.title, &landing_subject_source(&state)),
21116            m.title
21117        );
21118        // An operator's rename wins untouched.
21119        assert_eq!(
21120            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
21121            "feat: renamed by hand"
21122        );
21123    }
21124
21125    #[tokio::test]
21126    async fn github_text_rewrite_is_bounded_and_falls_back() {
21127        crate::run::pin_test_home();
21128        for (reply, accepted) in [
21129            (
21130                "TITLE: fix: retries\nAdd retries for failed requests.",
21131                true,
21132            ),
21133            (
21134                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
21135                false,
21136            ),
21137            (
21138                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
21139                true,
21140            ),
21141            (
21142                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
21143                true,
21144            ),
21145            (
21146                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
21147                true,
21148            ),
21149            (
21150                "TITLE: fix: retries\n日本語の説明です。 token=secret",
21151                false,
21152            ),
21153            ("TITLE: fix: retries\n", false),
21154        ] {
21155            let dir = tempfile::tempdir().unwrap();
21156            let mut runner = runner_at(RunStatus::Gating);
21157            runner.state = state_with_summary(
21158                "add retries",
21159                "TITLE: fix: retries\n日本語の説明を書きます。",
21160            );
21161            runner.state.repo = dir.path().to_owned();
21162            runner.state.candidates[0].worktree = dir.path().to_owned();
21163            let mut author = spec("alpha");
21164            author.command = vec![
21165                "sh".into(),
21166                "-c".into(),
21167                "printf '%s' \"$REWRITE_REPLY\"".into(),
21168            ];
21169            author.env.insert("REWRITE_REPLY".into(), reply.into());
21170            runner.state.config.agents = vec![author];
21171            seed_public_summary(&mut runner);
21172            let winner = runner.state.candidates[0].clone();
21173            let first = runner
21174                .guarded_pr_message(&winner, None, true)
21175                .await
21176                .unwrap();
21177            let message = if accepted {
21178                first.expect("a passing rewrite never asks")
21179            } else {
21180                // Withheld: the run parks on a question and posts nothing yet.
21181                assert!(first.is_none());
21182                assert!(runner.state.parked);
21183                let store = ask::Questions::open();
21184                let asked: Vec<_> = store
21185                    .list()
21186                    .into_iter()
21187                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
21188                    .collect();
21189                assert_eq!(asked.len(), 1);
21190                assert!(!asked[0].detail.contains("secret"));
21191                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
21192                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
21193                assert_eq!(
21194                    asked[0]
21195                        .choices
21196                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
21197                    rewrite.is_some()
21198                );
21199                // Silence: the question lapses and the neutral text is used,
21200                // without a second rewrite call or a second question.
21201                store
21202                    .update(&asked[0].id, |q| {
21203                        q.abandon("test");
21204                        Ok(())
21205                    })
21206                    .unwrap();
21207                let again = runner
21208                    .guarded_pr_message(&winner, None, true)
21209                    .await
21210                    .unwrap()
21211                    .expect("an abandoned question falls back");
21212                let n = store
21213                    .list()
21214                    .into_iter()
21215                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
21216                    .count();
21217                assert_eq!(n, 1, "asked once per rejected text");
21218                again
21219            };
21220            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
21221            assert!(!message.body.contains("secret"));
21222            assert!(!message.body.contains("prefix"));
21223            assert!(!message.title.contains("secret"));
21224            assert_eq!(
21225                message.body.contains("Add retries for failed requests."),
21226                accepted
21227            );
21228            assert_eq!(
21229                runner.state.seats["github-text-rewrite"].turns, 1,
21230                "only one rewrite invocation"
21231            );
21232            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
21233            if accepted {
21234                assert!(runner.state.github_text.is_none());
21235                assert!(!runner.state.parked);
21236                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
21237                assert!(message.body.contains("## Final verification"));
21238                assert!(message.body.contains("Evidence: final diff"));
21239                assert!(
21240                    message
21241                        .body
21242                        .contains(&format!("magi:run/{}", runner.state.id))
21243                );
21244            }
21245            if !accepted {
21246                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
21247                assert!(
21248                    message
21249                        .body
21250                        .contains(&format!("magi:run/{}", runner.state.id))
21251                );
21252            }
21253        }
21254    }
21255
21256    fn scripted(id: &str, reply: &str) -> AgentSpec {
21257        let mut a = spec(id);
21258        a.command = vec![
21259            "sh".into(),
21260            "-c".into(),
21261            "printf '%s' \"$REWRITE_REPLY\"".into(),
21262        ];
21263        a.env.insert("REWRITE_REPLY".into(), reply.into());
21264        a
21265    }
21266
21267    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
21268    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
21269
21270    /// Gate tests inject generated public output, not internal reports.
21271    fn seed_public_summary(runner: &mut Runner) {
21272        let repo = &runner.state.repo;
21273        let g = |args: &[&str]| {
21274            let out = std::process::Command::new("git")
21275                .args(args)
21276                .current_dir(repo)
21277                .quiet()
21278                .output()
21279                .unwrap();
21280            assert!(out.status.success(), "git {args:?}");
21281            String::from_utf8_lossy(&out.stdout).trim().to_owned()
21282        };
21283        g(&["init", "-q", "-b", "main"]);
21284        g(&[
21285            "-c",
21286            "user.name=Test",
21287            "-c",
21288            "user.email=test@example.invalid",
21289            "commit",
21290            "-q",
21291            "--allow-empty",
21292            "-m",
21293            "feat: fixture",
21294        ]);
21295        let head = g(&["rev-parse", "HEAD"]);
21296        g(&["update-ref", "refs/remotes/origin/main", &head]);
21297        let evidence = crate::pr_outcome::summary_evidence(
21298            &runner.state,
21299            crate::pr_outcome::FinalHead::At(&head),
21300        );
21301        let generated = &runner.state.candidates[0].summary;
21302        runner.state.change_summary = Some(crate::run::ChangeSummary {
21303            base: head.clone(),
21304            head,
21305            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
21306            title: summary_title(generated).unwrap(),
21307            body: summary_without_title(generated),
21308        });
21309    }
21310
21311    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
21312        let mut runner = runner_at(RunStatus::Gating);
21313        runner.state = state_with_summary("add retries", JA_SUMMARY);
21314        if review_only {
21315            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21316        }
21317        runner.state.repo = dir.to_owned();
21318        runner.state.candidates[0].worktree = dir.to_owned();
21319        runner.state.config.agents = agents;
21320        seed_public_summary(&mut runner);
21321        runner
21322    }
21323
21324    fn gate_questions(runner: &Runner) -> usize {
21325        ask::Questions::open()
21326            .list()
21327            .into_iter()
21328            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
21329            .count()
21330    }
21331
21332    #[tokio::test]
21333    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
21334        crate::run::pin_test_home();
21335        let dir = tempfile::tempdir().unwrap();
21336        let mut runner = rewrite_runner(
21337            dir.path(),
21338            true,
21339            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
21340        );
21341        runner.state.config.roles.language_judge =
21342            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
21343        let winner = runner.state.candidates[0].clone();
21344        let msg = runner
21345            .guarded_pr_message(&winner, None, true)
21346            .await
21347            .unwrap()
21348            .expect("an English rewrite posts without asking");
21349        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
21350        assert!(msg.body.contains("<details>kept</details>"));
21351        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
21352        assert_eq!(gate_questions(&runner), 0);
21353        assert!(runner.state.github_text.is_none());
21354        assert!(!runner.state.seats.contains_key("impl-A"));
21355        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21356    }
21357
21358    #[tokio::test]
21359    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
21360        crate::run::pin_test_home();
21361        let dir = tempfile::tempdir().unwrap();
21362        let mut runner = rewrite_runner(
21363            dir.path(),
21364            false,
21365            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
21366        );
21367        runner.state.config.roles.language_judge =
21368            Some(crate::config::AgentChoice::One("beta".into()));
21369        runner.state.seats.insert(
21370            "impl-A".into(),
21371            SeatState::new("impl-A", "alpha", runner.state.seed),
21372        );
21373        let winner = runner.state.candidates[0].clone();
21374        let msg = runner
21375            .guarded_pr_message(&winner, None, true)
21376            .await
21377            .unwrap()
21378            .expect("rewritten");
21379        assert!(msg.body.contains("<details>kept</details>"));
21380        assert_eq!(runner.state.seats["impl-A"].turns, 0);
21381        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21382        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
21383    }
21384
21385    #[tokio::test]
21386    async fn rewrite_reuses_a_resumable_implementer_session() {
21387        crate::run::pin_test_home();
21388        let dir = tempfile::tempdir().unwrap();
21389        let mut runner = rewrite_runner(
21390            dir.path(),
21391            false,
21392            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
21393        );
21394        runner.state.config.roles.language_judge =
21395            Some(crate::config::AgentChoice::One("beta".into()));
21396        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
21397        seat.turns = 1;
21398        runner.state.seats.insert("impl-A".into(), seat);
21399        let winner = runner.state.candidates[0].clone();
21400        runner
21401            .guarded_pr_message(&winner, None, true)
21402            .await
21403            .unwrap()
21404            .expect("rewritten");
21405        assert_eq!(runner.state.seats["impl-A"].turns, 2);
21406        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
21407    }
21408
21409    #[tokio::test]
21410    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
21411        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
21412        // session, was out of quota, and nothing else was asked.
21413        crate::run::pin_test_home();
21414        let dir = tempfile::tempdir().unwrap();
21415        let mut exhausted = spec("alpha");
21416        exhausted.command = vec!["false".into()];
21417        let mut runner = rewrite_runner(
21418            dir.path(),
21419            false,
21420            vec![exhausted, scripted("beta", EN_REWRITE)],
21421        );
21422        // The implementer's own agent is in the role too: it is not asked twice.
21423        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
21424            "alpha".into(),
21425            "beta".into(),
21426        ]));
21427        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
21428        seat.turns = 1;
21429        runner.state.seats.insert("impl-A".into(), seat);
21430        let winner = runner.state.candidates[0].clone();
21431        let msg = runner
21432            .guarded_pr_message(&winner, None, true)
21433            .await
21434            .unwrap()
21435            .expect("the role rewrites after the implementer fails");
21436        assert!(msg.body.contains("`src/net.rs`"));
21437        assert_eq!(
21438            runner.state.seats["impl-A"].turns, 2,
21439            "the implementer was tried once"
21440        );
21441        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
21442        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21443    }
21444
21445    #[tokio::test]
21446    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
21447        crate::run::pin_test_home();
21448        let dir = tempfile::tempdir().unwrap();
21449        let mut broken = spec("bad");
21450        broken.command = vec!["false".into()];
21451        let mut runner = rewrite_runner(
21452            dir.path(),
21453            true,
21454            vec![
21455                broken,
21456                scripted("noformat", "Add retries."),
21457                scripted("good", EN_REWRITE),
21458            ],
21459        );
21460        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
21461            "bad".into(),
21462            "noformat".into(),
21463            "good".into(),
21464        ]));
21465        let winner = runner.state.candidates[0].clone();
21466        let msg = runner
21467            .guarded_pr_message(&winner, None, true)
21468            .await
21469            .unwrap()
21470            .expect("the third agent rewrites");
21471        assert!(msg.body.contains("`src/net.rs`"));
21472        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
21473        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
21474    }
21475
21476    #[tokio::test]
21477    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
21478        crate::run::pin_test_home();
21479        // (agents, role): still Japanese; no TITLE; nothing resolvable.
21480        for (agents, role, saved) in [
21481            (
21482                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
21483                Some("beta"),
21484                true,
21485            ),
21486            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
21487            (Vec::new(), None, false),
21488        ] {
21489            let dir = tempfile::tempdir().unwrap();
21490            let mut runner = rewrite_runner(dir.path(), true, agents);
21491            runner.state.config.roles.language_judge =
21492                role.map(|r| crate::config::AgentChoice::One(r.into()));
21493            let winner = runner.state.candidates[0].clone();
21494            let first = runner
21495                .guarded_pr_message(&winner, None, true)
21496                .await
21497                .unwrap();
21498            assert!(first.is_none(), "withheld text parks the run");
21499            assert!(runner.state.parked);
21500            assert_eq!(gate_questions(&runner), 1);
21501            let g = runner.state.github_text.as_ref().unwrap();
21502            assert_eq!(g.rewrite.is_some(), saved);
21503            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
21504        }
21505    }
21506
21507    #[tokio::test]
21508    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
21509        crate::run::pin_test_home();
21510        let dir = tempfile::tempdir().unwrap();
21511        let mut runner = runner_at(RunStatus::Gating);
21512        runner.state =
21513            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
21514        runner.state.repo = dir.path().to_owned();
21515        runner.state.candidates[0].worktree = dir.path().to_owned();
21516        let mut author = spec("alpha");
21517        author.command = vec![
21518            "sh".into(),
21519            "-c".into(),
21520            "printf '%s' \"$REWRITE_REPLY\"".into(),
21521        ];
21522        author.env.insert(
21523            "REWRITE_REPLY".into(),
21524            format!(
21525                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
21526                "English sentence. ".repeat(600)
21527            ),
21528        );
21529        runner.state.config.agents = vec![author];
21530        seed_public_summary(&mut runner);
21531        let winner = runner.state.candidates[0].clone();
21532        assert!(
21533            runner
21534                .guarded_pr_message(&winner, None, true)
21535                .await
21536                .unwrap()
21537                .is_none()
21538        );
21539        let record = runner.state.github_text.clone().unwrap();
21540        let rewrite = record.rewrite.clone().unwrap();
21541        assert_eq!(rewrite.failed, ["body-language"]);
21542        assert!(
21543            rewrite
21544                .body
21545                .ends_with(&format!("magi:run/{}", runner.state.id))
21546        );
21547        let store = ask::Questions::open();
21548        let q = store.get(record.question.as_ref().unwrap()).unwrap();
21549        assert!(
21550            q.choices
21551                .contains(&crate::github_text::USE_REWRITE.to_owned())
21552        );
21553        for label in [
21554            "ORIGINAL",
21555            "FALLBACK",
21556            "REWRITE",
21557            "Rules still failed: body-language",
21558            "truncated",
21559        ] {
21560            assert!(q.detail.contains(label), "{label}");
21561        }
21562        let artifact = std::fs::read_to_string(crate::run::artifact_path(
21563            &runner.state,
21564            &crate::github_text::artifact_name(&record.fingerprint),
21565        ))
21566        .unwrap();
21567        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
21568        // Restart from serialized state, then retry after a bad owner title.
21569        runner.state =
21570            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
21571        store
21572            .update(&q.id, |q| {
21573                q.say("まだ日本語")?;
21574                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
21575            })
21576            .unwrap();
21577        assert!(
21578            runner
21579                .guarded_pr_message(&winner, None, true)
21580                .await
21581                .unwrap()
21582                .is_none()
21583        );
21584        let retry = runner.state.github_text.as_ref().unwrap();
21585        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
21586        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
21587        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
21588        assert!(
21589            q.posting_comparison
21590                .as_ref()
21591                .unwrap()
21592                .matches_detail(&q.detail)
21593        );
21594        store
21595            .update(&q.id, |q| {
21596                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
21597            })
21598            .unwrap();
21599        let message = runner
21600            .guarded_pr_message(&winner, None, true)
21601            .await
21602            .unwrap()
21603            .unwrap();
21604        assert_eq!(message.title, rewrite.title);
21605        assert_eq!(message.body, rewrite.body);
21606        assert_eq!(
21607            runner.state.github_text.as_ref().unwrap().choice,
21608            Some(GithubTextChoice::Rewrite)
21609        );
21610        let again = runner
21611            .guarded_pr_message(&winner, None, true)
21612            .await
21613            .unwrap()
21614            .unwrap();
21615        assert_eq!(again.title, message.title);
21616        assert_eq!(again.body, message.body);
21617        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21618        assert!(
21619            !runner
21620                .state
21621                .events
21622                .iter()
21623                .any(|e| e.message.contains(&rewrite.title)
21624                    || e.message.contains("English sentence."))
21625        );
21626    }
21627
21628    #[tokio::test]
21629    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
21630        crate::run::pin_test_home();
21631        let dir = tempfile::tempdir().unwrap();
21632        let mut runner = runner_at(RunStatus::Gating);
21633        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
21634        runner.state.repo = dir.path().to_owned();
21635        runner.state.candidates[0].worktree = dir.path().to_owned();
21636        seed_public_summary(&mut runner);
21637        let winner = runner.state.candidates[0].clone();
21638        assert!(
21639            runner
21640                .guarded_pr_message(&winner, None, true)
21641                .await
21642                .unwrap()
21643                .is_none()
21644        );
21645        let record = runner.state.github_text.as_ref().unwrap();
21646        assert!(record.rewrite.is_none());
21647        // Records written before the additive field deserialize with no rewrite.
21648        let mut old = serde_json::to_value(record).unwrap();
21649        old.as_object_mut().unwrap().remove("rewrite");
21650        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
21651        assert!(old.rewrite.is_none());
21652        let id = record.question.clone().unwrap();
21653        ask::Questions::open()
21654            .update(&id, |q| {
21655                q.choices.push(crate::github_text::USE_REWRITE.into());
21656                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
21657            })
21658            .unwrap();
21659        let message = runner
21660            .guarded_pr_message(&winner, None, true)
21661            .await
21662            .unwrap()
21663            .unwrap();
21664        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
21665        assert_eq!(
21666            runner.state.github_text.as_ref().unwrap().choice,
21667            Some(GithubTextChoice::Fallback)
21668        );
21669    }
21670
21671    #[tokio::test]
21672    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
21673        crate::run::pin_test_home();
21674        let dir = tempfile::tempdir().unwrap();
21675        let mut runner = runner_at(RunStatus::Gating);
21676        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
21677        runner.state.repo = dir.path().to_owned();
21678        runner.state.candidates[0].worktree = dir.path().to_owned();
21679        seed_public_summary(&mut runner);
21680        let winner = runner.state.candidates[0].clone();
21681        assert!(
21682            runner
21683                .guarded_pr_message(&winner, None, true)
21684                .await
21685                .unwrap()
21686                .is_none()
21687        );
21688        let store = ask::Questions::open();
21689        let find = |run: &str| {
21690            store
21691                .list()
21692                .into_iter()
21693                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
21694                .unwrap()
21695        };
21696        let run = runner.state.id.clone();
21697        let q = find(&run);
21698        let fp = runner
21699            .state
21700            .github_text
21701            .as_ref()
21702            .unwrap()
21703            .fingerprint
21704            .clone();
21705        let name = crate::github_text::artifact_name(&fp);
21706        let path = crate::run::artifact_path(&runner.state, &name);
21707        assert!(q.detail.contains(&path.display().to_string()));
21708        assert!(q.detail.contains("title-language: title"));
21709        let saved = std::fs::read_to_string(&path).expect("artifact written");
21710        assert!(saved.contains("日本語のタイトル"));
21711        assert!(q.cwd.is_none());
21712        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
21713        // A bad replacement is refused and asked about once more.
21714        let say = |id: &str, text: &str, choice: &str| {
21715            store
21716                .update(id, |q| {
21717                    q.thread.push(ask::Turn {
21718                        who: ask::Who::Operator,
21719                        body: text.to_owned(),
21720                        at: jiff::Timestamp::now(),
21721                        note: None,
21722                    });
21723                    q.answer(ask::Answer::Choice(choice.to_owned()))
21724                })
21725                .unwrap();
21726        };
21727        say(&q.id, "まだ日本語", "use my text");
21728        assert!(
21729            runner
21730                .guarded_pr_message(&winner, None, true)
21731                .await
21732                .unwrap()
21733                .is_none()
21734        );
21735        let second = find(&run);
21736        assert_ne!(second.id, q.id);
21737        assert!(second.detail.contains("did not pass"));
21738        // The retry still compares the original text; the rejected
21739        // replacement is named by category only.
21740        assert!(second.detail.contains("日本語のタイトル"));
21741        assert!(!second.detail.contains("まだ日本語"));
21742        assert!(second.detail.contains(&path.display().to_string()));
21743        say(&second.id, "fix: retry failed requests", "use my text");
21744        let message = runner
21745            .guarded_pr_message(&winner, None, true)
21746            .await
21747            .unwrap()
21748            .expect("a vetted replacement is posted");
21749        assert_eq!(message.title, "fix: retry failed requests");
21750        assert_eq!(
21751            runner
21752                .state
21753                .github_text
21754                .as_ref()
21755                .unwrap()
21756                .chosen_title
21757                .as_deref(),
21758            Some("fix: retry failed requests")
21759        );
21760        assert!(runner.state.github_text.as_ref().unwrap().resolved);
21761        // Resumed again: the saved decision is applied, nothing is re-asked.
21762        let again = runner
21763            .guarded_pr_message(&winner, None, true)
21764            .await
21765            .unwrap()
21766            .unwrap();
21767        assert_eq!(again.title, message.title);
21768        assert_eq!(
21769            store
21770                .list()
21771                .into_iter()
21772                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
21773                .count(),
21774            2
21775        );
21776    }
21777
21778    #[test]
21779    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
21780        // What magi itself writes stays English under any configured language,
21781        // so a future localisation of these headings fails here. (The agents'
21782        // own text is also checked by the posting gate.)
21783        let mut state = state_with_summary(
21784            "add retries",
21785            "TITLE: fix(web): batch reads\n- reads run.json once",
21786        );
21787        state.config.graph.language = "ja".to_owned();
21788        let m = pr_message(&state, 'A');
21789        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
21790        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
21791
21792        // The task is the operator's own text: it goes in untouched, and the
21793        // fallback title (no summary) may be in its language too.
21794        let task = "今回やってほしいこと: results projector を直す";
21795        let mut state = state_with_summary(task, "- no title line");
21796        state.config.graph.language = "ja".to_owned();
21797        let m = pr_message(&state, 'A');
21798        assert_eq!(
21799            m.title,
21800            format!("chore: land candidate A of run {}", state.id)
21801        );
21802        assert!(
21803            m.body.contains(&format!(
21804                "<summary>Original task</summary>\n\n{task}\n\n</details>"
21805            )),
21806            "{}",
21807            m.body
21808        );
21809    }
21810
21811    #[test]
21812    fn pr_message_scrubs_home_paths_and_addresses() {
21813        let state = state_with_summary(
21814            "fix it in /Users/someone/src/x",
21815            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
21816        );
21817        let m = pr_message(&state, 'A');
21818        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
21819            assert!(!m.body.contains(leak), "{}", m.body);
21820        }
21821        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
21822    }
21823
21824    #[test]
21825    fn pr_message_survives_a_task_that_closes_details() {
21826        let state = state_with_summary("a </details> b", "TITLE: fix: x");
21827        let m = pr_message(&state, 'A');
21828        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
21829    }
21830
21831    #[test]
21832    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
21833        let cmd = manual_merge_command(
21834            MergeStyle::Squash,
21835            Path::new("/repo"),
21836            "b",
21837            "fix: \"quoted\" $(x) `y`\n\nbody",
21838        );
21839        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
21840    }
21841
21842    #[test]
21843    fn manual_merge_command_matches_the_configured_style() {
21844        let repo = Path::new("/repo");
21845        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
21846
21847        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
21848        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
21849
21850        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
21851        assert_eq!(
21852            squash,
21853            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
21854             \"Merge magi run 0832 (candidate A)\""
21855        );
21856
21857        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
21858        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
21859    }
21860
21861    #[test]
21862    fn a_nudge_gets_a_quarter_of_the_budget() {
21863        // The judge and implement budgets magi ships with.
21864        assert_eq!(retry_budget(secs(1200), true), secs(300));
21865        assert_eq!(retry_budget(secs(3600), true), secs(900));
21866    }
21867
21868    #[test]
21869    fn a_resent_prompt_keeps_the_whole_budget() {
21870        // The seat kept no context, so the retry is the original job again and
21871        // shortening it would only guarantee a second failure.
21872        assert_eq!(retry_budget(secs(1200), false), secs(1200));
21873        assert_eq!(retry_budget(secs(60), false), secs(60));
21874    }
21875
21876    #[test]
21877    fn the_floor_never_exceeds_the_original_budget() {
21878        // A short configured timeout must not be *raised* by the floor: the
21879        // operator asked for a bound, and a retry may not outlast the attempt
21880        // it is retrying.
21881        assert_eq!(retry_budget(secs(60), true), secs(60));
21882        assert_eq!(retry_budget(secs(480), true), secs(120));
21883        assert_eq!(retry_budget(secs(0), true), secs(0));
21884    }
21885
21886    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
21887        agent::CommandEvidence {
21888            id: "item1".to_owned(),
21889            description: "cargo test".to_owned(),
21890            exit_code,
21891            result_summary: String::new(),
21892            source: "codex".to_owned(),
21893        }
21894    }
21895
21896    #[test]
21897    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
21898        // No evidence is not the same fact as unconfirmed evidence: a
21899        // backend with no adapter, or a reply that ran no commands at all,
21900        // must not be misread as carrying a dangling job.
21901        assert!(!has_unconfirmed_command(&[]));
21902    }
21903
21904    #[test]
21905    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
21906        // Deliberately not a check on the exit code's *value*: a fixer
21907        // legitimately runs something that fails mid-iteration before it
21908        // succeeds, and that must never by itself reopen a valid report.
21909        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
21910        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
21911        assert!(!has_unconfirmed_command(&[
21912            evidence(Some(0)),
21913            evidence(Some(101))
21914        ]));
21915    }
21916
21917    #[test]
21918    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
21919        assert!(has_unconfirmed_command(&[
21920            evidence(Some(0)),
21921            evidence(None)
21922        ]));
21923    }
21924
21925    #[test]
21926    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
21927        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21928        assert_eq!(
21929            verified_noop_claim(true, &[], text).as_deref(),
21930            Some("already fixed by b32cfc4, on main.")
21931        );
21932    }
21933
21934    #[test]
21935    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
21936        // A timeout or a bad exit code reads as the ordinary loss it is,
21937        // whatever the reply's own prose claims.
21938        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21939        assert!(verified_noop_claim(false, &[], text).is_none());
21940    }
21941
21942    #[test]
21943    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
21944        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21945        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
21946        // A confirmed command alongside the marker is fine.
21947        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
21948    }
21949
21950    #[test]
21951    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
21952        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
21953    }
21954
21955    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
21956    /// pair, in order, labelled A, B, C, ...
21957    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
21958        runner.state.candidates = shape
21959            .iter()
21960            .enumerate()
21961            .map(|(i, &(empty, verified))| Candidate {
21962                index: i,
21963                label: (b'A' + i as u8) as char,
21964                agent: "sonnet".to_owned(),
21965                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
21966                worktree: PathBuf::from(format!("/wt/{i}")),
21967                summary: String::new(),
21968                stat: String::new(),
21969                files: 0,
21970                commits: 0,
21971                empty,
21972                failed: None,
21973                verified_noop: verified.map(str::to_owned),
21974                duration_ms: 0,
21975                folded: false,
21976            })
21977            .collect();
21978    }
21979
21980    #[test]
21981    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
21982        ask_test_home();
21983        let mut runner = runner_at(RunStatus::Implementing);
21984        set_candidates(
21985            &mut runner,
21986            &[
21987                (true, Some("already on main at b32cfc4")),
21988                (true, Some("same fix, see the existing test")),
21989            ],
21990        );
21991
21992        runner
21993            .after_implement()
21994            .expect("a verified no-op is not an error");
21995
21996        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
21997    }
21998
21999    #[test]
22000    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
22001        ask_test_home();
22002        let mut runner = runner_at(RunStatus::Implementing);
22003        // Candidate A declares a verified no-op; candidate B simply wrote
22004        // nothing and said nothing about why. One candidate's claim is not
22005        // the whole run's agreement.
22006        set_candidates(
22007            &mut runner,
22008            &[(true, Some("already on main at b32cfc4")), (true, None)],
22009        );
22010
22011        let err = runner
22012            .after_implement()
22013            .expect_err("an unverified empty candidate must still fail the run");
22014
22015        assert!(
22016            err.to_string().contains("no candidate produced a change"),
22017            "{err}"
22018        );
22019        assert_eq!(runner.state.status, RunStatus::Failed);
22020    }
22021
22022    #[test]
22023    fn after_implement_still_fails_an_ordinary_all_empty_run() {
22024        ask_test_home();
22025        let mut runner = runner_at(RunStatus::Implementing);
22026        set_candidates(&mut runner, &[(true, None), (true, None)]);
22027
22028        let err = runner
22029            .after_implement()
22030            .expect_err("no candidate declared anything; this is an ordinary failure");
22031
22032        assert!(
22033            err.to_string().contains("no candidate produced a change"),
22034            "{err}"
22035        );
22036        assert_eq!(runner.state.status, RunStatus::Failed);
22037    }
22038}
22039
22040/// A graft rollback whose restore could not be confirmed (see
22041/// [`Runner::graft`]).
22042#[derive(Debug)]
22043struct GraftRestoreFailed(String);
22044
22045impl std::fmt::Display for GraftRestoreFailed {
22046    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
22047        write!(
22048            f,
22049            "could not restore the winner to its pre-graft commit: {}",
22050            self.0
22051        )
22052    }
22053}
22054
22055impl std::error::Error for GraftRestoreFailed {}
22056
22057/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
22058/// not the agent: they never draw on an agent's own timeout, and running out of
22059/// them never reads as an agent failure.
22060#[cfg(test)]
22061mod resource_wait_tests {
22062    use super::*;
22063    use std::collections::BTreeMap;
22064
22065    fn sh(id: &str, script: &str) -> AgentSpec {
22066        AgentSpec {
22067            id: id.to_owned(),
22068            kind: crate::config::AgentKind::Command,
22069            model: None,
22070            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
22071            extra_args: Vec::new(),
22072            env: BTreeMap::new(),
22073            prompt_delivery: None,
22074            max_turn_secs: None,
22075        }
22076    }
22077
22078    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
22079        SeatJob {
22080            seat: SeatState::new("impl-A", &spec.id, 1),
22081            spec,
22082            cwd: dir.to_path_buf(),
22083            prompt: "do it".to_owned(),
22084            timeout,
22085            allow_write: true,
22086            sessions: false,
22087            artifacts: dir.join("art"),
22088            stem: "impl-A".to_owned(),
22089            handover: None,
22090        }
22091    }
22092
22093    fn state(resource_wait_secs: u64) -> RunState {
22094        let mut config = Config::default();
22095        config.graph.timeout_resource_wait = resource_wait_secs;
22096        RunState::new(
22097            PathBuf::from("/repo"),
22098            "main".to_owned(),
22099            "abc1234".to_owned(),
22100            "task".to_owned(),
22101            config,
22102        )
22103    }
22104
22105    fn hold(cache: &Path) -> crate::cache::Guard {
22106        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
22107        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
22108            crate::cache::AcquireOutcome::Acquired(g) => g,
22109            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
22110        }
22111    }
22112
22113    async fn run(
22114        jobs: Vec<SeatJob>,
22115        sem: Arc<Semaphore>,
22116        cache: Option<&Path>,
22117        st: &mut RunState,
22118    ) -> Vec<(usize, SeatState, AgentOutcome)> {
22119        let prompts = Prompts::default();
22120        let ctx = WaveCtx {
22121            run: "r",
22122            node: "implement",
22123            prompts: &prompts,
22124            cache,
22125            round: None,
22126            carry_seats: false,
22127            pauses: None,
22128        };
22129        wave(jobs, sem, &ctx, st, 0).await
22130    }
22131
22132    #[tokio::test]
22133    async fn verification_cache_wait_preserves_the_command_budget() {
22134        crate::run::pin_test_home();
22135        let tmp = tempfile::tempdir().unwrap();
22136        let cache = tmp.path().join("target");
22137        let guard = hold(&cache);
22138        let release = tokio::spawn(async move {
22139            tokio::time::sleep(Duration::from_millis(100)).await;
22140            guard.release();
22141        });
22142        let mut st = state(10);
22143        let budget = Duration::from_millis(50);
22144        let wait = repository_wait(&st, None);
22145        let (outcomes, _) = with_cache_lease(
22146            &mut st,
22147            Some(&cache),
22148            "gate",
22149            "gate",
22150            tmp.path(),
22151            "abc1234",
22152            budget,
22153            "test gate",
22154            &wait,
22155            |_, available| async move {
22156                assert_eq!(
22157                    available, budget,
22158                    "contention must not shorten verification"
22159                );
22160                (Vec::new(), false, Vec::new())
22161            },
22162        )
22163        .await
22164        .unwrap();
22165        release.await.unwrap();
22166        assert!(outcomes.is_empty(), "{outcomes:?}");
22167        assert!(
22168            st.events
22169                .iter()
22170                .any(|event| event.message.contains("waiting for"))
22171        );
22172        assert!(st.resource_wait.is_none());
22173        // The verification body no longer owns the lease after returning.
22174        hold(&cache).release();
22175    }
22176
22177    #[tokio::test]
22178    async fn verification_cache_wait_expires_without_starting_commands() {
22179        crate::run::pin_test_home();
22180        let tmp = tempfile::tempdir().unwrap();
22181        let cache = tmp.path().join("target");
22182        let _held = hold(&cache);
22183        let mut st = state(0);
22184        let wait = repository_wait(&st, None);
22185        let (outcomes, _) = with_cache_lease(
22186            &mut st,
22187            Some(&cache),
22188            "gate",
22189            "gate",
22190            tmp.path(),
22191            "abc1234",
22192            Duration::from_secs(60),
22193            "test gate",
22194            &wait,
22195            |_, _| async { panic!("commands must not start without the cache lease") },
22196        )
22197        .await
22198        .unwrap();
22199        assert_eq!(outcomes.len(), 1);
22200        assert!(outcomes[0].resource_blocked);
22201        assert!(st.resource_wait.is_none());
22202        assert!(matches!(
22203            st.resource_block.unwrap().cause,
22204            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
22205        ));
22206    }
22207
22208    #[tokio::test]
22209    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
22210        crate::run::pin_test_home();
22211        let tmp = tempfile::tempdir().unwrap();
22212        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
22213        let cache = tmp.path().join("cache");
22214        let mut state = state(21600);
22215        let wait = repository_wait(&state, None);
22216        let (outcomes, _) = with_cache_lease(
22217            &mut state,
22218            Some(&cache),
22219            "gate",
22220            "gate",
22221            tmp.path(),
22222            "head",
22223            Duration::from_secs(60),
22224            "gate",
22225            &wait,
22226            |_, _| async { panic!("freshness failure must not start verification") },
22227        )
22228        .await
22229        .unwrap();
22230        assert_eq!(outcomes.len(), 1);
22231        assert!(outcomes[0].resource_blocked);
22232        let block = state.resource_block.unwrap();
22233        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
22234        assert!(!block.description().contains("timed out"));
22235    }
22236
22237    #[tokio::test]
22238    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
22239        crate::run::pin_test_home();
22240        let tmp = tempfile::tempdir().unwrap();
22241        let cache = tmp.path().join("cache");
22242        let _held = hold(&cache);
22243        let mut state = state(21600);
22244        let pause = Pause::new();
22245        let interrupt = Pause::new();
22246        let cause = ParkCause::Operator;
22247        pause.park_for(cause.clone());
22248        let prompts = Prompts::default();
22249        let ctx = WaveCtx {
22250            run: "test",
22251            node: "fix",
22252            prompts: &prompts,
22253            cache: Some(&cache),
22254            round: None,
22255            carry_seats: false,
22256            pauses: Some((&pause, &interrupt)),
22257        };
22258        let result = wave(
22259            vec![job(
22260                sh("writer", "exit 99"),
22261                tmp.path(),
22262                Duration::from_secs(1),
22263            )],
22264            Arc::new(Semaphore::new(1)),
22265            &ctx,
22266            &mut state,
22267            0,
22268        )
22269        .await;
22270        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
22271        assert_eq!(result[0].1.turns, 0);
22272        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
22273        assert_eq!(
22274            state.resource_block.unwrap().cause,
22275            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
22276        );
22277        assert!(state.active.is_empty());
22278    }
22279
22280    #[tokio::test]
22281    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
22282        crate::run::pin_test_home();
22283        let tmp = tempfile::tempdir().unwrap();
22284        let cache = tmp.path().join("target-a");
22285        let guard = hold(&cache);
22286        // Held for longer than the agent's whole 1s budget.
22287        tokio::spawn(async move {
22288            tokio::time::sleep(Duration::from_millis(1500)).await;
22289            guard.release();
22290        });
22291        let mut st = state(60);
22292        let results = run(
22293            vec![job(
22294                sh("alpha", "echo ok"),
22295                tmp.path(),
22296                Duration::from_secs(1),
22297            )],
22298            Arc::new(Semaphore::new(1)),
22299            Some(&cache),
22300            &mut st,
22301        )
22302        .await;
22303        assert!(
22304            matches!(results[0].2, AgentOutcome::Ok(_)),
22305            "the wait must not be charged to the agent"
22306        );
22307        assert!(st.resource_block.is_none());
22308        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
22309        assert!(st.resource_wait.is_none() && st.active.is_empty());
22310    }
22311
22312    #[tokio::test]
22313    #[cfg(unix)]
22314    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
22315        use std::os::fd::AsRawFd as _;
22316        crate::run::pin_test_home();
22317        let tmp = tempfile::tempdir().unwrap();
22318        crate::git::git(tmp.path(), &["init"]).await.unwrap();
22319        let common = tmp.path().join(".git");
22320        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
22321        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
22322        assert_eq!(
22323            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
22324            0
22325        );
22326        let mut st = state(1);
22327        st.config.coordination = crate::config::test_coordination();
22328        let mut reader = job(
22329            sh("reader", "echo must-not-run"),
22330            tmp.path(),
22331            Duration::from_secs(30),
22332        );
22333        reader.allow_write = false;
22334        let results = run(
22335            vec![reader.clone()],
22336            Arc::new(Semaphore::new(1)),
22337            None,
22338            &mut st,
22339        )
22340        .await;
22341        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
22342        assert_eq!(results[0].1.turns, 0);
22343        assert!(FailClass::of(&results[0].2).is_none());
22344        assert_eq!(
22345            st.resource_block.as_ref().unwrap().kind,
22346            "repository coordination"
22347        );
22348        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
22349        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
22350        drop(external);
22351        let editor = crate::repo_lock::SeatLease::for_checkout(
22352            tmp.path(),
22353            true,
22354            crate::config::test_coordination().as_ref(),
22355        )
22356        .await
22357        .unwrap();
22358        let pause = Pause::new();
22359        let interrupt = Pause::new();
22360        let prompts = Prompts::default();
22361        let ctx = WaveCtx {
22362            run: "r",
22363            node: "review",
22364            prompts: &prompts,
22365            cache: None,
22366            round: None,
22367            carry_seats: false,
22368            pauses: Some((&pause, &interrupt)),
22369        };
22370        let (results, ()) = tokio::join!(
22371            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
22372            async {
22373                tokio::time::sleep(Duration::from_millis(50)).await;
22374                pause.park();
22375            }
22376        );
22377        assert!(
22378            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
22379        );
22380        assert_eq!(results[0].1.turns, 0);
22381        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
22382        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
22383        drop(editor);
22384    }
22385
22386    #[tokio::test]
22387    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
22388        crate::run::pin_test_home();
22389        let tmp = tempfile::tempdir().unwrap();
22390        let cache = tmp.path().join("target-b");
22391        let _held = hold(&cache);
22392        let mut st = state(1);
22393        let results = run(
22394            vec![job(
22395                sh("alpha", "echo ok"),
22396                tmp.path(),
22397                Duration::from_secs(1),
22398            )],
22399            Arc::new(Semaphore::new(1)),
22400            Some(&cache),
22401            &mut st,
22402        )
22403        .await;
22404        let (_, seat, out) = &results[0];
22405        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
22406        assert!(wave_blocked(&results));
22407        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
22408        assert_eq!(seat.turns, 0, "nothing was asked");
22409        assert!(
22410            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
22411            "no agent was spawned"
22412        );
22413        assert!(FailClass::of(out).is_none(), "not a failure class");
22414        let block = st.resource_block.as_ref().expect("recorded for the resume");
22415        assert_eq!(block.kind, "shared build cache");
22416        assert!(matches!(
22417            block.cause,
22418            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
22419        ));
22420        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
22421        assert!(st.active.is_empty());
22422    }
22423
22424    #[tokio::test]
22425    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
22426        crate::run::pin_test_home();
22427        let tmp = tempfile::tempdir().unwrap();
22428        let mut st = state(1);
22429        let results = run(
22430            vec![job(
22431                sh("alpha", "echo ok"),
22432                tmp.path(),
22433                Duration::from_secs(1),
22434            )],
22435            Arc::new(Semaphore::new(0)),
22436            None,
22437            &mut st,
22438        )
22439        .await;
22440        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
22441        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
22442        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
22443    }
22444
22445    #[tokio::test]
22446    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
22447        crate::run::pin_test_home();
22448        let tmp = tempfile::tempdir().unwrap();
22449        let mut st = state(60);
22450        let results = run(
22451            vec![job(
22452                sh("alpha", "sleep 30"),
22453                tmp.path(),
22454                Duration::from_secs(1),
22455            )],
22456            Arc::new(Semaphore::new(1)),
22457            None,
22458            &mut st,
22459        )
22460        .await;
22461        let out = &results[0].2;
22462        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
22463        let class = FailClass::of(out).expect("a failure");
22464        assert_eq!(class, FailClass::Timeout);
22465        assert!(should_hand_over(None, &class), "existing handover policy");
22466        assert!(st.resource_block.is_none());
22467    }
22468}