Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim shared by targeted repairs and repository reassignment.
534/// Held before asynchronous fix preflight and released on every return or panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549pub(crate) struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Capture task identity and repository assignment before the first
773        // save, so publication scope survives crashes and later task moves.
774        state.fetched_base = Some(fetched_base);
775        state.set_origin(origin)?;
776        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777        state.status = RunStatus::Prep;
778        state.event("start", format!("run {} created", state.id));
779        startup.finish(&mut state, Ok(()))?;
780        Ok(Self {
781            state,
782            roles,
783            sem: Arc::new(Semaphore::new(max_parallel)),
784            pause: Pause::new(),
785            interrupt: Pause::new(),
786        })
787    }
788
789    /// Open a review-only run against work that already exists on `branch`.
790    ///
791    /// The expensive half of the graph is the implement wave — measured at
792    /// 111 and 134 internal tool-loop turns on this repository, against a
793    /// handful for a judge or a reviewer. The cheap half is worth running on
794    /// hand-written work too, and there was no way to reach it.
795    ///
796    /// No new state and no schema change are needed: a run with **one** viable
797    /// candidate and a tally already decided degrades `execute` to exactly
798    /// review → gate → merge, because `judge` skips a single-candidate field,
799    /// `deliberate` has fewer than two first choices to reconcile, `vote`
800    /// returns early, `tally` is already present and `fold_losers` has no
801    /// losers. Resuming such a run therefore does the right thing as well.
802    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803        Self::review_taking_over(repo, branch, config, None, origin).await
804    }
805
806    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
807    /// at the same task still has `branch` checked out, its worktree is
808    /// released first if that is safe (see [`crate::handover`]), and the
809    /// review refuses with the reason if it is not. `None` is a hand-run
810    /// review: it has no earlier attempts, so only a worktree of a dead run
811    /// magi recorded itself can be released.
812    pub async fn review_taking_over(
813        repo: &Path,
814        branch: &str,
815        config: Config,
816        takeover: Option<crate::handover::Takeover>,
817        origin: Origin,
818    ) -> Result<Self> {
819        let repo = git::toplevel(repo).await?;
820        let missing = agent::missing_programs(&config.agents);
821        if !missing.is_empty() {
822            bail!(
823                "these agent programs are not on PATH: {}. Fix the roster in \
824                 magi.toml or install them.",
825                missing.join(", ")
826            );
827        }
828        let base_branch =
829            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830                .await?;
831        if base_branch == branch {
832            bail!("`{branch}` is the base branch; there is nothing to review against");
833        }
834        let fetched_base = resolve_base(
835            &repo,
836            &base_branch,
837            &config.merge.remote,
838            config.coordination.as_ref(),
839        )
840        .await?;
841        let base_commit = fetched_base.commit.clone();
842
843        let roles = config.resolve_roles()?;
844        let max_parallel = config.graph.max_parallel.max(1);
845        let mut state = RunState::new(
846            repo.clone(),
847            base_branch,
848            base_commit.clone(),
849            format!("Review the work already on branch `{branch}`."),
850            config,
851        );
852        state.fetched_base = Some(fetched_base);
853        state.set_origin(origin)?;
854        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856        let opened = async {
857            startup.stage(&mut state, "handover")?;
858
859            // Released before anything else touches the branch: a stale local
860            // branch is moved with `git branch -f`, which git refuses while an
861            // earlier attempt's worktree still has it checked out. Everything
862            // after this point that can fail puts the old run back.
863            // A hand-run review has no task, hence no earlier attempts, but a
864            // worktree of a dead run magi made may still be released.
865            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866                earlier: Vec::new(),
867                home: crate::run::home(),
868                choice: None,
869                task_instruction: None,
870            });
871            let wait = repository_wait(&state, None);
872            let coordination_config = state.config.coordination.clone();
873            let coordination =
874                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875                    .await?;
876            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877            if let Some(released) = &released {
878                state.event(
879                    "release",
880                    format!(
881                        "took `{branch}` over from run {}: its worktree was released: {}",
882                        crate::run::short_of(&released.old_id),
883                        released.audit
884                    ),
885                );
886            }
887            let opened = async {
888                startup.stage(&mut state, "review-preflight")?;
889                // The owner's answer to an earlier divergence question is applied
890                // here: after the release (git will not move a checked-out branch)
891                // and before the sync that would otherwise ask again.
892                state.foreign_checkout = crate::handover::preflight_review(
893                    &repo,
894                    branch,
895                    &takeover.home,
896                    &state.config.merge.remote,
897                )
898                .await?;
899                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901                }
902                if let Some(choice) = takeover.choice.as_ref()
903                    && let Err(e) =
904                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905                            .await
906                {
907                    return Err(e.context("applying the owner's answer about the diverged branch"));
908                }
909                // An earlier attempt of this task that published the task head, and
910                // whose work is the branch under review, hands that head over: the
911                // review run updates the same pull request. Without one (a legacy
912                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
913                // own head and its own open pull request is adopted as before.
914                for id in takeover.earlier.iter().rev() {
915                    if let Ok(old) = RunState::load_under(id, &takeover.home)
916                        && let Some(head) = old.pr_head.clone()
917                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918                    {
919                        state.pr_head = Some(head);
920                        break;
921                    }
922                }
923                drop(coordination);
924                startup.stage(&mut state, "open-review")?;
925                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926            }
927            .await;
928            if opened.is_err()
929                && let Some(released) = &released
930            {
931                // Restoration is ownership checked and best effort. Keep the original
932                // startup cause even if reacquiring coordination is no longer possible.
933                match crate::repo_lock::SeatLease::operation_with(
934                    &repo,
935                    coordination_config.as_ref(),
936                    &wait,
937                )
938                .await
939                {
940                    Ok(_coordination) => {
941                        released.restore(&repo, branch).await;
942                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
943                            .map(|old| {
944                                format!(
945                                    "predecessor released_to={:?}; candidate path exists={}",
946                                    old.released_to,
947                                    old.candidates.iter().any(|c| {
948                                        c.branch == branch && c.worktree.exists()
949                                    }),
950                                )
951                            })
952                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954                    }
955                    Err(error) => {
956                        tracing::warn!(
957                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958                            state.id
959                        );
960                        state.event(
961                            "startup-restore",
962                            format!("restoration not attempted: {error:#}"),
963                        );
964                    }
965                }
966            }
967            opened
968        }
969        .await;
970        startup.finish(&mut state, opened)?;
971        Ok(Self {
972            state,
973            roles,
974            sem: Arc::new(Semaphore::new(max_parallel)),
975            pause: Pause::new(),
976            interrupt: Pause::new(),
977        })
978    }
979
980    /// Recover exactly the operator-selected public implementation, without touching old sources.
981    pub async fn recover_pr(
982        repo: &Path,
983        task: &str,
984        selection: crate::pr_recovery::Selection,
985        mut config: Config,
986    ) -> Result<Self> {
987        selection.configure(&mut config)?;
988        let assignment = crate::run::task_assignment_for(task)?;
989        selection.check_task_assignment(task, &selection.instruction, assignment)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.set_origin(crate::run::Origin::queue(task))?;
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize one bounded synchronization interval, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false, None).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true, None).await
1357    }
1358
1359    pub(crate) async fn authorize_base_recovery_control(
1360        home: &Path,
1361        task: &str,
1362        control: &mut crate::base_recovery_control::Transaction,
1363    ) -> Result<()> {
1364        let id = control.request.run.clone();
1365        Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366    }
1367
1368    async fn authorize_base_recovery_request(
1369        home: &Path,
1370        id: &str,
1371        task: &str,
1372        resumption: bool,
1373        mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374    ) -> Result<()> {
1375        let _owner = crate::pr_head::owner_lock(home, id)?;
1376        let mut state = RunState::load_under(id, home)?;
1377        crate::pr_head::ensure_owner(&state)?;
1378        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379        anyhow::ensure!(
1380            queued.execution_runs().last().is_some_and(|run| run == id)
1381                && queued.status != crate::queue::TaskStatus::Done
1382                && !queued.fresh_start
1383                && queued.review_branch.is_none(),
1384            "run task ownership changed"
1385        );
1386        anyhow::ensure!(
1387            state
1388                .origin
1389                .as_ref()
1390                .and_then(|o| o.task.as_deref())
1391                .is_none_or(|owner| owner == task),
1392            "run belongs to another task"
1393        );
1394        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395        anyhow::ensure!(
1396            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397            "run is live or its ownership is unknown"
1398        );
1399        // Judge the previous driver's lease before acquiring our maintenance
1400        // lease. The owner lock excludes new driver claims across both steps.
1401        let _execution = crate::driver::maintenance(home, id)?;
1402        anyhow::ensure!(
1403            state.status.resumable()
1404                && state.released_to.is_none()
1405                && state.retired_to.is_none()
1406                && state.pr_taken_over_by.is_none(),
1407            "run ownership is final"
1408        );
1409        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410        anyhow::ensure!(
1411            !state.pr_publication_pending
1412                && !crate::handover::landing_checkpoint(&state)
1413                && state.merge_decision.is_none(),
1414            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415        );
1416        let winner = state.winner().context("run has no retained winner")?;
1417        anyhow::ensure!(
1418            git::is_clean(&winner.worktree).await?,
1419            "retained worktree has uncommitted changes"
1420        );
1421        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422        anyhow::ensure!(
1423            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424            "retained worktree does not match the winner branch"
1425        );
1426        if let Some(control) = control.as_ref() {
1427            anyhow::ensure!(
1428                head.starts_with(&control.request.head),
1429                "retained HEAD changed since the owner request"
1430            );
1431            anyhow::ensure!(
1432                state
1433                    .base_sync_recovery
1434                    .stop
1435                    .as_ref()
1436                    .is_none_or(|stop| stop.head == head),
1437                "retained HEAD differs from the recorded stop head; inspect it first"
1438            );
1439        }
1440        state.ensure_base_episode();
1441        state.record_base_sync_stop(&head);
1442        anyhow::ensure!(
1443            state
1444                .base_sync_recovery
1445                .stop
1446                .as_ref()
1447                .is_some_and(|s| s.kind == crate::base_sync_recovery::StopKind::Exhausted),
1448            "base recovery refused: missing eligible movement-exhaustion evidence; inspect with `magi task inspect-base-stop {task} --run {id}`. Textual stops require an evidence-bound base-conflict grant"
1449        );
1450        let fetched =
1451            git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1452        anyhow::ensure!(
1453            fetched.ok(),
1454            "cannot authorize base recovery without a fetched base"
1455        );
1456        let approved_tip = git::rev_parse(
1457            &state.repo,
1458            &format!("{}/{}", state.config.merge.remote, state.base_branch),
1459        )
1460        .await?;
1461        if let Some(control) = control.as_deref_mut() {
1462            let queue = crate::queue::Queue::at(home.join("queue"));
1463            control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1464            if let Some(prior) = state
1465                .base_sync_recovery
1466                .authorizations
1467                .iter()
1468                .find(|a| a.request.as_deref() == Some(&control.key()))
1469            {
1470                anyhow::ensure!(
1471                    !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1472                    "recovery request was consumed or invalidated; it cannot queue again"
1473                );
1474                return control.enqueue(&queue);
1475            }
1476        }
1477        if resumption {
1478            anyhow::ensure!(
1479                crate::daemon::compatible_run_instruction(&state, &queued),
1480                "resumption instruction does not match the retained run"
1481            );
1482            let approval = queued
1483                .resumption
1484                .as_ref()
1485                .context("no operator resumption")?;
1486            anyhow::ensure!(
1487                approval.instruction == queued.instruction,
1488                "resumption approval does not match the current task instruction"
1489            );
1490            state.authorize_base_sync_resumption(task, approval)?;
1491        } else {
1492            state.authorize_base_sync(task)?;
1493        }
1494        let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1495        state
1496            .base_sync_recovery
1497            .authorizations
1498            .last_mut()
1499            .unwrap()
1500            .scope = Some(crate::base_sync_recovery::Scope {
1501            tip: approved_tip,
1502            instruction: queued.instruction.clone(),
1503            episode,
1504        });
1505        if let Some(control) = control.as_deref_mut() {
1506            let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1507            authorization.request = Some(control.key());
1508            authorization.generation = control.generation();
1509            authorization.decision = control.decision();
1510            authorization.question = control
1511                .request
1512                .decision
1513                .as_ref()
1514                .map(|_| control.request.request.clone());
1515        }
1516        state.event(
1517            "base-recovery",
1518            "owner authorized a bounded base synchronization interval; fresh verification required",
1519        );
1520        state.save_under(home)?;
1521        if let Some(control) = control {
1522            control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1523        }
1524        Ok(())
1525    }
1526
1527    /// Reopen an existing run.
1528    pub fn resume(id: &str) -> Result<Self> {
1529        let state = RunState::load(id)?;
1530        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1531        if state.startup.as_ref().is_some_and(|s| !s.complete)
1532            || (state.status == RunStatus::Prep
1533                && state.instruction.trim().is_empty()
1534                && state.candidates.is_empty()
1535                && state.prep_complete == Some(false))
1536        {
1537            bail!(
1538                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1539            );
1540        }
1541        crate::pr_head::ensure_owner(&state)?;
1542        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1543            bail!(
1544                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1545            );
1546        }
1547        if let Some(to) = &state.released_to {
1548            bail!(
1549                "run {} cannot be resumed: its worktree was released to run {}",
1550                state.short(),
1551                crate::run::short_of(to)
1552            );
1553        }
1554        let roles = state.config.resolve_roles()?;
1555        if let Some(id) = &state.config.graph.selected_implementer
1556            && state.candidates.iter().any(|c| c.agent != *id)
1557        {
1558            bail!(
1559                "replacement implementer pin does not match the saved candidates; task must stay held"
1560            );
1561        }
1562        let max_parallel = state.config.graph.max_parallel.max(1);
1563        Ok(Self {
1564            state,
1565            roles,
1566            sem: Arc::new(Semaphore::new(max_parallel)),
1567            pause: Pause::new(),
1568            interrupt: Pause::new(),
1569        })
1570    }
1571
1572    /// Walk the graph to a terminal state, skipping nodes already recorded.
1573    ///
1574    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1575    /// the phone - ends here, so this is the one place a run that ended
1576    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1577    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1578    pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1579        // Allocate before polling: boxing inside an async wrapper would still
1580        // retain its construction frame while the nested graph is polled.
1581        Box::pin(self.execute_inner())
1582    }
1583
1584    async fn execute_inner(&mut self) -> Result<()> {
1585        anyhow::ensure!(
1586            self.state.startup.as_ref().is_none_or(|s| s.complete),
1587            "startup incomplete; inspect retained evidence before explicit recovery"
1588        );
1589        if let Some(selection) = &self.state.pr_recovery
1590            && (selection.source_proof.is_some() || selection.source_run.is_some())
1591        {
1592            Box::pin(selection.verify_source(&self.state.repo)).await?;
1593        }
1594        // Queue claims cover daemon work; this also serializes manual drivers
1595        // consuming a repair answer, including the new review and decision.
1596        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1597            || self.state.land_approval.is_some()
1598            || self.state.merge_decision.is_some()
1599            || !self.state.merge_repairs.is_empty()
1600            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1601        {
1602            let claim = FixClaim::acquire(&self.state.dir())?;
1603            crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1604            let disk = RunState::load(&self.state.id)?;
1605            let checkpoint = |s: &RunState| {
1606                (
1607                    s.merge_repairs.len(),
1608                    s.merge_repairs
1609                        .last()
1610                        .map(|r| (r.question.clone(), r.phase)),
1611                    s.land_approval
1612                        .as_ref()
1613                        .map(|a| (a.question.clone(), a.head.clone())),
1614                )
1615            };
1616            if checkpoint(&disk) != checkpoint(&self.state) {
1617                bail!(
1618                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1619                );
1620            }
1621            Some(claim)
1622        } else {
1623            None
1624        };
1625        let mut execution = self.claim_driver().await?;
1626        let runner = &mut *execution.runner;
1627        let wait = runner.repository_wait();
1628        // Keep the graph's large future off the callers' stack, including
1629        // standalone task attempts running on a default-sized test thread.
1630        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1631        let result = match result {
1632            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1633                record_repository_block(&mut runner.state, &e);
1634                runner.park_here()?;
1635                Ok(())
1636            }
1637            other => other,
1638        };
1639        execution.driver.finish(&mut runner.state).await?;
1640        let ended = if result.is_err() {
1641            Some({
1642                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1643                if let Err(error) = &result {
1644                    notice.message = format!(
1645                        "Run {} stopped with an error: {}",
1646                        runner.state.id,
1647                        error.to_string().lines().next().unwrap_or("unknown error")
1648                    );
1649                }
1650                notice
1651            })
1652        } else {
1653            crate::notices::run_ended(&runner.state)
1654        };
1655        if let Some(notice) = ended {
1656            crate::notices::raise(notice);
1657        }
1658        result
1659    }
1660
1661    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1662        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1663    }
1664
1665    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1666        let wait = self.repository_wait();
1667        coordinate_operation(&mut self.state, cwd, &wait).await
1668    }
1669
1670    /// Validate ownership and persist the driver claim in one critical section.
1671    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1672        let owner = match crate::pr_head::owner_lock_wait(
1673            &crate::run::home(),
1674            &self.state.id,
1675            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1676        )
1677        .await
1678        {
1679            Ok(owner) => owner,
1680            Err(error) => {
1681                let detail = format!("worktree maintenance or ownership update: {error:#}");
1682                // Admission did not own run.json: retain this resource outcome
1683                // in memory for queue settlement without overwriting a concurrent
1684                // retirement or publication transaction.
1685                self.state.resource_block = Some(crate::run::ResourceBlock {
1686                    cause: crate::run::ResourceBlockCause::Other,
1687                    node: "start".into(),
1688                    kind: "worktree maintenance".into(),
1689                    detail: detail.clone(),
1690                    at: Timestamp::now(),
1691                });
1692                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1693            }
1694        };
1695        let disk = RunState::load(&self.state.id)?;
1696        crate::pr_head::ensure_owner(&disk)?;
1697        let claimed =
1698            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1699        // Only proven-live ownership refuses here. Unknown records (legacy runs
1700        // with no claim marker or process identity) fall through to the
1701        // exclusive execution lease taken just below, which is what actually
1702        // keeps a second driver out.
1703        anyhow::ensure!(
1704            disk.liveness(claimed) != crate::run::Liveness::Live,
1705            "run already has a live driver; wait for its execution to end"
1706        );
1707        if let Some(to) = &disk.released_to {
1708            bail!(
1709                "run {} cannot continue: its worktree was released to run {}",
1710                self.state.short(),
1711                crate::run::short_of(to)
1712            );
1713        }
1714        anyhow::ensure!(
1715            serde_json::to_value(&disk.base_sync_recovery)?
1716                == serde_json::to_value(&self.state.base_sync_recovery)?,
1717            "base recovery changed since this run was loaded; reload the saved run"
1718        );
1719        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1720        let mut execution = DriverExecution {
1721            runner: self,
1722            driver,
1723        };
1724        // Declare after the execution guard: on any claim error this short
1725        // lock must drop before that guard reloads and releases ownership.
1726        let _owner = owner;
1727        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1728            .await?;
1729        let state = &mut execution.runner.state;
1730        // Moving again, so it is no longer parked. Set before the walk rather
1731        // than in `resume`, so every way of re-entering the graph clears it
1732        // and a card cannot claim a run is waiting to be resumed while the
1733        // agents are already working.
1734        state.parked = false;
1735        state.resource_wait = None;
1736        state.resource_block = None;
1737        // Any seat this state still lists as answering belongs to whatever
1738        // process last drove this run — this one included, if it crashed
1739        // mid-wave. Cleared and flushed immediately, before anything else
1740        // runs, so a resume can never show a seat as live when nothing is
1741        // asking it anything yet; the node that actually dispatches the next
1742        // wave repopulates it.
1743        state.clear_active();
1744        // Recorded in the same spot, and flushed together with the clear
1745        // above: this is the pid a reader checks (`RunState::liveness`) when
1746        // no daemon claim exists to answer "is a process still driving this
1747        // run" — a plain `magi run` / `magi review` typed into a terminal
1748        // claims nothing there. Always overwritten, never only-if-absent, so
1749        // a resumed run's stale pid from a previous, possibly-dead process
1750        // can never survive into this one's own report. Unlike
1751        // `clear_active`, this changes on every single `execute()` call, so
1752        // the save below is now unconditional rather than only-if-cleared.
1753        //
1754        // `driver_started_at` is recorded in the same breath, from this same
1755        // pid, so `liveness` can tell a live pid that is genuinely still us
1756        // apart from one the OS has since handed to an unrelated process —
1757        // see that field's own doc for why the pid alone is not enough.
1758        let pid = std::process::id();
1759        state.driver_pid = Some(pid);
1760        state.driver_started_at = crate::proc::process_started_at(pid);
1761        state.driver_exited = false;
1762        state.driver_claimed = Some(true);
1763        state.begin_pass();
1764        execution.driver.bind(state);
1765        state.save()?;
1766        execution.driver.record(state)?;
1767        Ok(execution)
1768    }
1769
1770    /// No conflict checkpoint proves dispatch was never reached. End that
1771    /// consumed generation so an owner can explicitly authorize a fresh one.
1772    async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1773        let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1774        if let Err(error) = &result
1775            && !error.is::<crate::repo_lock::ResourceBlocked>()
1776            && self
1777                .state
1778                .repair_grants
1779                .last()
1780                .is_some_and(|g| g.conflict.is_none())
1781        {
1782            self.state.repair_grants.last_mut().unwrap().phase =
1783                crate::repair_budget::Phase::Complete;
1784            self.state.status = RunStatus::Blocked;
1785            self.state.save()?;
1786            bail!(
1787                "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1788                self.state.passes.last().map_or(0, |p| p.index)
1789            );
1790        }
1791        result
1792    }
1793
1794    /// A grant owns only these repair attempts. Confirmation reviews cannot
1795    /// create another fixer allowance, and every operation is restart-checkpointed.
1796    async fn execute_granted_repair(&mut self) -> Result<()> {
1797        use crate::repair_budget::{Kind, Phase};
1798        let index = self.state.repair_grants.len() - 1;
1799        let winner = self
1800            .state
1801            .winner()
1802            .cloned()
1803            .context("repair winner missing")?;
1804        loop {
1805            let grant = self.state.repair_grants[index].clone();
1806            match grant.phase {
1807                Phase::Pending => {
1808                    if grant.spent >= grant.receipt.request.rounds {
1809                        self.state.repair_grants[index].phase = Phase::Complete;
1810                        self.state.status = RunStatus::Blocked;
1811                        self.state.save()?;
1812                        return Ok(());
1813                    }
1814                    // Reserve before dispatch. A crash can consume an attempt,
1815                    // but can never duplicate it or reset saved fixer history.
1816                    self.state.repair_grants[index].spent += 1;
1817                    self.state.repair_grants[index].phase = Phase::Fixing;
1818                    self.state.save()?;
1819                    match grant.receipt.request.kind {
1820                        Kind::BaseConflict => self.repair_base_conflict(true).await?,
1821                        Kind::Gate => {
1822                            let failed = self.state.gate.clone();
1823                            let _ = self.gate_fix_round(&winner, &failed).await?;
1824                        }
1825                        Kind::Review => {
1826                            let mut record = self
1827                                .state
1828                                .reviews
1829                                .last()
1830                                .cloned()
1831                                .context("review failure missing")?;
1832                            let round = record.round;
1833                            let step = self.run_fix_round(&winner, &mut record).await?;
1834                            self.state.reviews[round - 1] = record;
1835                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1836                                self.state.repair_grants[index].phase = Phase::Complete;
1837                                self.state.status = RunStatus::Blocked;
1838                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1839                                self.state.save()?;
1840                                return Ok(());
1841                            }
1842                            if matches!(step, FixStep::Resource) {
1843                                self.stop_for_resource()?;
1844                                return Ok(());
1845                            }
1846                            self.state.save()?;
1847                        }
1848                    }
1849                    if self.resource_halt()? {
1850                        return Ok(());
1851                    }
1852                }
1853                Phase::Fixing => {
1854                    if grant.receipt.request.kind == Kind::BaseConflict {
1855                        self.repair_base_conflict(false).await?;
1856                    }
1857                    // Interrupted fixers may have left source changes or an
1858                    // unpushed commit. Preserve them before fresh assessment.
1859                    let _coordination = self.coordinate(&winner.worktree).await?;
1860                    let rescued = git::rescue_commit(
1861                        &winner.worktree,
1862                        "magi: preserve interrupted granted repair",
1863                    )
1864                    .await?;
1865                    self.state.note_withheld("repair-grant", &rescued.withheld);
1866                    // Fresh checks must supersede even a same-head failure.
1867                    // Archive rather than erase failure evidence or counters.
1868                    self.state
1869                        .verification_history
1870                        .push(crate::run::VerificationEvidence {
1871                            superseded_by: None,
1872                            archived_at: Some(Timestamp::now()),
1873                            fixes_spent: Some(self.state.gate_fixes.len()),
1874                            gate_head: self.state.gate_head.take(),
1875                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1876                            gate: std::mem::take(&mut self.state.gate),
1877                            e2e: self.state.head_e2e.take(),
1878                        });
1879                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1880                        self.state
1881                            .verification_history
1882                            .push(crate::run::VerificationEvidence {
1883                                superseded_by: None,
1884                                archived_at: Some(Timestamp::now()),
1885                                fixes_spent: Some(self.state.gate_fixes.len()),
1886                                gate_head: None,
1887                                gate_ran: false,
1888                                gate: Vec::new(),
1889                                e2e: Some(e2e),
1890                            });
1891                    }
1892                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1893                    self.state.repair_grants[index].head =
1894                        git::rev_parse(&winner.worktree, "HEAD").await?;
1895                    self.state.repair_grants[index].phase = Phase::Reviewing;
1896                    if grant.receipt.request.kind == Kind::BaseConflict {
1897                        self.state.base_sync_recovery.stop = None;
1898                        if let Some(sync) = &mut self.state.base_sync {
1899                            sync.tip = grant
1900                                .receipt
1901                                .request
1902                                .tip
1903                                .clone()
1904                                .context("repair base tip missing")?;
1905                            sync.conflict = None;
1906                            sync.behind = 0;
1907                        }
1908                        refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1909                    }
1910                    self.state.save()?;
1911                }
1912                Phase::Reviewing => {
1913                    self.review_loop_pass().await?;
1914                    if self.resource_halt()? {
1915                        return Ok(());
1916                    }
1917                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1918                    let clean = self.state.reviews.len() > grant.review_start
1919                        && self.state.reviews.last().is_some_and(|r| {
1920                            r.head == head
1921                                && r.clean
1922                                && r.expected > 0
1923                                && r.answered == r.expected
1924                                && matches!(
1925                                    r.e2e_status(),
1926                                    E2eStatus::Passed | E2eStatus::NotConfigured
1927                                )
1928                        });
1929                    if !clean {
1930                        // A confirmation review carries no additional fix budget.
1931                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1932                            == Kind::Review
1933                            && grant.spent < grant.receipt.request.rounds
1934                            && self.state.reviews.last().is_some_and(|r| {
1935                                r.fix.is_none()
1936                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1937                            }) {
1938                            Phase::Pending
1939                        } else {
1940                            Phase::Complete
1941                        };
1942                        self.state.status = RunStatus::Blocked;
1943                        self.state.event(
1944                            "repair-grant",
1945                            "fresh review did not clear the repaired head; no publication",
1946                        );
1947                        self.state.save()?;
1948                        if self.state.repair_grants[index].phase == Phase::Pending {
1949                            continue;
1950                        }
1951                        return Ok(());
1952                    }
1953                    self.state.repair_grants[index].phase = Phase::Gating;
1954                    self.state.save()?;
1955                }
1956                Phase::Gating => {
1957                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1958                    self.state.reconcile_verification_head(&head);
1959                    let wait = self.repository_wait();
1960                    if self
1961                        .state
1962                        .reviews
1963                        .last()
1964                        .is_none_or(|r| r.head != head || !r.clean)
1965                    {
1966                        self.state.status = RunStatus::Blocked;
1967                        self.state.repair_grants[index].phase = Phase::Complete;
1968                        self.state.event(
1969                            "repair-grant",
1970                            "head changed since fresh review; repair held",
1971                        );
1972                        self.state.save()?;
1973                        return Ok(());
1974                    }
1975                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1976                        return Ok(());
1977                    }
1978                    let outcomes = self.run_gate(&winner).await?;
1979                    if verify_inconclusive(&outcomes) {
1980                        return Ok(());
1981                    }
1982                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1983                    if after != head {
1984                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1985                        self.state.repair_grants[index].head = after;
1986                        self.state.repair_grants[index].phase = Phase::Reviewing;
1987                        self.state.status = RunStatus::Blocked;
1988                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1989                        self.state.save()?;
1990                        return Ok(());
1991                    }
1992                    let passed = outcomes.iter().all(CommandOutcome::ok);
1993                    // Follow-up grants owe the same guarded checkout delivery
1994                    // as the conflict repair that introduced the new head.
1995                    if passed
1996                        && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1997                    {
1998                        return Ok(());
1999                    }
2000                    self.state.repair_grants[index].succeeded = passed;
2001                    self.state.gate = outcomes;
2002                    self.state.gate_ran = true;
2003                    self.state.status = if passed {
2004                        RunStatus::Gating
2005                    } else {
2006                        RunStatus::Blocked
2007                    };
2008                    self.state.repair_grants[index].phase = if !passed
2009                        && grant.receipt.request.kind == Kind::Gate
2010                        && grant.spent < grant.receipt.request.rounds
2011                    {
2012                        Phase::Pending
2013                    } else {
2014                        Phase::Complete
2015                    };
2016                    if passed && self.state.config.merge.mode == MergeMode::Pr {
2017                        // Persist publication debt with the successful checkpoint.
2018                        self.state.pr_publication_pending = true;
2019                    } else if passed {
2020                        self.state.status = RunStatus::Ready;
2021                    }
2022                    self.state.save()?;
2023                    if !passed {
2024                        continue;
2025                    }
2026                    // Existing PR ownership and push lease stay in place. The
2027                    // normal publisher owes the new reviewed/gated SHA, and
2028                    // land's fresh owner approval remains mandatory.
2029                    if self.state.config.merge.mode != MergeMode::Pr {
2030                        self.state.status = RunStatus::Ready;
2031                        self.state.event(
2032                            "repair-grant",
2033                            "repair verified locally; grant carries no merge authorization",
2034                        );
2035                        self.state.save()?;
2036                        return Ok(());
2037                    }
2038                    return self.merge().await;
2039                }
2040                Phase::Complete => return Ok(()),
2041            }
2042        }
2043    }
2044
2045    async fn execute_graph(&mut self) -> Result<()> {
2046        // Even a terminal run may reenter through an explicit recovery action.
2047        // Restore before prep shortcuts, saved conversations or graph nodes use
2048        // a disposable path; GC may already have removed its original objects.
2049        let archive_home = crate::run::home().canonicalize()?;
2050        if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2051            let repo = self.state.repo.clone();
2052            let _coordination = self.coordinate(&repo).await?;
2053            ensure_setup_config(&mut self.state, &repo).await?;
2054            crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2055        }
2056        self.state.configure_recovery_resume()?;
2057        if self
2058            .state
2059            .repair_grants
2060            .last()
2061            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2062        {
2063            return self.execute_granted_repair().await;
2064        }
2065        if self
2066            .state
2067            .repair_grants
2068            .last()
2069            .is_some_and(|g| !g.succeeded)
2070        {
2071            self.state.status = RunStatus::Blocked;
2072            self.state.save()?;
2073            return Ok(());
2074        }
2075        // Preservation may invalidate an old empty classification. Resolve it
2076        // before any resume shortcut can reuse a verdict or publish a branch.
2077        self.persist_candidates().await?;
2078        // A saved base stop outranks stage markers, including legacy Voting.
2079        // Recovery never reenters implementation, deliberation or voting.
2080        if self.state.base_sync_blocked() {
2081            self.sync_to_base().await?;
2082            if !self.state.base_sync_blocked()
2083                && self.state.status != RunStatus::AlreadyInBase
2084                && !self.resource_halt()?
2085            {
2086                self.finish_after_tally().await?;
2087            }
2088            return Ok(());
2089        }
2090        if self
2091            .state
2092            .merge_decision
2093            .as_ref()
2094            .is_some_and(|decision| !decision.closed)
2095            && self.state.status != RunStatus::Landing
2096        {
2097            if crate::merge_decision::resumable_hold(&self.state) {
2098                self.state.merge = Some(MergeOutcome {
2099                    mode: MergeMode::Pr,
2100                    ok: false,
2101                    detail: self
2102                        .state
2103                        .merge_decision
2104                        .as_ref()
2105                        .unwrap()
2106                        .target
2107                        .pr
2108                        .clone(),
2109                    empty: false,
2110                });
2111                self.run_land().await?;
2112                self.settle_questions();
2113                return Ok(());
2114            }
2115            bail!(
2116                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2117            );
2118        }
2119        if self.state.config.graph.selected_implementer.is_some()
2120            && self.state.status == RunStatus::Blocked
2121            && self.state.tally.is_none()
2122            && self.state.candidate_preservation.is_empty()
2123            && self.state.interrupted_implementations.is_empty()
2124        {
2125            bail!(
2126                "selected implementation is held; explicitly release the owning task to retry it"
2127            );
2128        }
2129
2130        if self.state.merge_repairs.last().is_some_and(|r| {
2131            matches!(
2132                r.phase,
2133                crate::run::MergeRepairPhase::Requested
2134                    | crate::run::MergeRepairPhase::Fixing
2135                    | crate::run::MergeRepairPhase::Reviewing
2136            )
2137        }) {
2138            if self.repair_merge_review().await? {
2139                self.run_land().await?;
2140            }
2141            self.settle_questions();
2142            return Ok(());
2143        }
2144        // Publication already passed review and gate. Resume only that node:
2145        // syncing to a newer base here could change the verified commit.
2146        if self.state.pr_publication_pending {
2147            self.merge().await?;
2148            if self.resource_halt()? {
2149                return Ok(());
2150            }
2151            self.state.save()?;
2152            self.settle_questions();
2153            return Ok(());
2154        }
2155        // A run that already lost its quorum never resumes into the verdict
2156        // machinery: `deliberate` and `vote` would otherwise clobber the
2157        // stalled marker back to Voting and the run would keep going past a
2158        // verdict that is no longer trustworthy. Everything already recorded is
2159        // kept, so the run stays resumable (or foldable) for a human to pick up.
2160        //
2161        // On --resume the run gets one chance to repair itself: the seats a
2162        // rate limit took out are re-asked. If their quota has since reset and
2163        // the quorum is restored, the run picks up and finishes; otherwise it
2164        // stays stale and still-resumable for a later retry. If it does not
2165        // recover, the returned status stays `Stalled` and nothing was
2166        // clobbered (the recovery only mutates entries for the lost seats).
2167        if self.state.status == RunStatus::Stalled {
2168            if self.recover_stall().await? {
2169                self.finish_after_tally().await?;
2170            } else {
2171                // Still below quorum: persist the marker and stay resumable.
2172                self.state.save()?;
2173            }
2174            return Ok(());
2175        }
2176        // A run parked inside `land` - watching CI, mid fix-round, or
2177        // waiting on the owner's merge approval - resumes directly into it,
2178        // never back through `prep`. Everything before `merge` already
2179        // concluded; that is the only way `status` reaches `Landing` in the
2180        // first place. Re-walking `review_loop` first would also be actively
2181        // wrong: its own status recomputation (see its doc) treats any
2182        // clean round as reason to set `status` to `Gating`, which would
2183        // clobber this marker before `merge` ever ran, and this run would
2184        // never find its way back into `land` at all.
2185        if self.state.status == RunStatus::Landing
2186            || crate::handover::landing_checkpoint(&self.state)
2187        {
2188            self.state.land_sync_pending = true;
2189            self.state.save()?;
2190            self.run_land().await?;
2191            // `run_land` may have settled the run right here - CI came back
2192            // green and the PR merged, say - without ever passing back
2193            // through `merge`'s own trailing call. Whatever it left `status`
2194            // as is what this has to read.
2195            self.settle_questions();
2196            return Ok(());
2197        }
2198        // A run parked at `merge` on the owner's word about a withheld PR text
2199        // resumes straight into `merge`. Re-walking the tail would sync to the
2200        // base again (possibly rebasing, possibly spending the fixer) after the
2201        // gate had passed, and then push a tree nobody reviewed or gated.
2202        if self.state.github_text.is_some()
2203            && self.state.merge.is_none()
2204            && self.state.status == RunStatus::Gating
2205        {
2206            self.merge().await?;
2207            if self.resource_halt()? {
2208                return Ok(());
2209            }
2210            self.state.save()?;
2211            return Ok(());
2212        }
2213        if let Err(e) = self.prep().await {
2214            if e.is::<crate::repo_lock::ResourceBlocked>() {
2215                return Err(e);
2216            }
2217            self.state.status = RunStatus::Blocked;
2218            self.state
2219                .prep_failure
2220                .get_or_insert_with(|| format!("{e:#}"));
2221            self.state
2222                .event("prep", format!("preparation stopped: {e:#}"));
2223            self.state.save()?;
2224            return Err(e);
2225        }
2226        if self.park_here()? {
2227            return Ok(());
2228        }
2229        self.advise().await?;
2230        if self.park_here()? {
2231            return Ok(());
2232        }
2233        self.implement().await?;
2234        if self.park_here()? {
2235            return Ok(());
2236        }
2237        // `after_implement` already saved the state and settled any open
2238        // questions when it set this; nothing later in the graph has
2239        // anything to judge.
2240        if self.state.status == RunStatus::VerifiedNoop {
2241            return Ok(());
2242        }
2243        self.judge().await?;
2244        if self.park_here()? {
2245            return Ok(());
2246        }
2247        self.deliberate().await?;
2248        if self.park_here()? {
2249            return Ok(());
2250        }
2251        self.vote().await?;
2252        if self.park_here()? {
2253            return Ok(());
2254        }
2255        self.tally()?;
2256        // A verdict that lost its quorum is not trustworthy: do not review,
2257        // gate, or merge on it. Everything already done is kept, so the run
2258        // stays resumable (or foldable); the human can replace the agent that
2259        // ran out of quota and pick it up.
2260        if self.state.status == RunStatus::Stalled {
2261            // Persist the stalled marker now — the normal end-of-execute save
2262            // below is below this early return, and without it a resumed run
2263            // would reload a pre-tally status and keep going.
2264            self.state.save()?;
2265            return Ok(());
2266        }
2267        self.finish_after_tally().await?;
2268        Ok(())
2269    }
2270
2271    /// A wave could not start its agents because a shared resource stayed busy
2272    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2273    /// resumable. Unstarted seats have no result; completed peers retain
2274    /// their saved candidate state.
2275    fn stop_for_resource(&mut self) -> Result<()> {
2276        // Resource acquisition observes cancellation inside a node. Complete
2277        // the same durable park as every other node boundary before returning.
2278        if self.pause.parked() || self.interrupt.parked() {
2279            self.park_here()?;
2280            return Ok(());
2281        }
2282        if self.state.resource_block.is_some() {
2283            self.state.status = RunStatus::Blocked;
2284        }
2285        self.state.save()
2286    }
2287
2288    /// Stop walking the graph if a node ended on a resource block.
2289    fn resource_halt(&mut self) -> Result<bool> {
2290        if self.state.resource_block.is_none() {
2291            return Ok(false);
2292        }
2293        self.stop_for_resource()?;
2294        Ok(true)
2295    }
2296
2297    /// Park here if asked to, recording it in the run's own timeline.
2298    ///
2299    /// Returns whether the caller should stop walking the graph. The state is
2300    /// saved either way by the node that just finished; this adds the event so
2301    /// the operator's card says why a run that is neither finished nor moving
2302    /// is sitting where it is.
2303    fn park_here(&mut self) -> Result<bool> {
2304        let parking = self.pause.parked() || self.interrupt.parked();
2305        if !parking && self.state.resource_block.is_some() {
2306            self.stop_for_resource()?;
2307            return Ok(true);
2308        }
2309        // Either handle asking is enough - see `Pause`'s own doc for why
2310        // they are never the same one. `interrupt` is checked second so a
2311        // reason it carries is preferred in the message below over a plain
2312        // shutdown park racing it at the same boundary.
2313        if !self.pause.parked() && !self.interrupt.parked() {
2314            return Ok(false);
2315        }
2316        if self.state.parked {
2317            return Ok(true);
2318        }
2319        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2320        if let Some(block) = &mut self.state.resource_block
2321            && matches!(
2322                block.cause,
2323                crate::run::ResourceBlockCause::Cancelled { .. }
2324            )
2325        {
2326            block.cause = crate::run::ResourceBlockCause::Cancelled {
2327                park: cause.clone(),
2328            };
2329        }
2330        let why = match cause.as_ref().map(ParkCause::label) {
2331            Some(reason) => format!(
2332                "parked after `{}` ({reason}) — resume to carry on from here",
2333                self.state.status.as_str()
2334            ),
2335            None => format!(
2336                "parked after `{}` — resume to carry on from here",
2337                self.state.status.as_str()
2338            ),
2339        };
2340        self.state.event("park", why);
2341        self.state
2342            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2343        self.state.parked = true;
2344        self.state.save()?;
2345        Ok(true)
2346    }
2347
2348    /// Only a new explicit operator approval may retry failed preparation.
2349    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2350        if self.state.prep_failure.is_none() {
2351            return Ok(());
2352        }
2353        let failed_at = self
2354            .state
2355            .events
2356            .iter()
2357            .rev()
2358            .find(|event| event.node == "prep")
2359            .map_or(self.state.updated_at, |event| event.at);
2360        anyhow::ensure!(
2361            approval_at > failed_at,
2362            "preparation retry requires a new explicit release after the failure"
2363        );
2364        self.state.prep_failure = None;
2365        self.state.prep_complete = Some(false);
2366        self.state
2367            .event("prep-retry", "operator approved retry of preparation");
2368        Ok(())
2369    }
2370
2371    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2372    pub fn on_pause(&mut self, pause: Pause) {
2373        self.pause = pause;
2374    }
2375
2376    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2377    /// scheduler asking this one run - and no other - to park so a task
2378    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2379    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2380    /// handle.
2381    pub fn watch_interrupt(&mut self, pause: Pause) {
2382        self.interrupt = pause;
2383    }
2384
2385    /// Abandon this run's own open questions, once `status` has actually
2386    /// settled rather than merely paused.
2387    ///
2388    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2389    /// either back up with the candidates, the review round and the seat
2390    /// sessions already on disk, so a question an implementer asked mid-round
2391    /// may still get a real answer read by a real resume. Only the statuses
2392    /// `resumable` excludes are actually final: the run merged, it reached
2393    /// `Ready` with nothing left to do, it failed outright with no
2394    /// established point to continue from, or every candidate agreed, with
2395    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2396    /// every one of those the seat that asked is gone for good, exactly like
2397    /// the run being deleted under `magi run rm` - so the same cleanup
2398    /// applies, worded for what actually happened instead of "the run was
2399    /// deleted".
2400    ///
2401    /// Best-effort and silent on success: called from every place `status`
2402    /// can land on one of those three, including ones a resumed run revisits,
2403    /// so it must cost nothing when there was nothing open to begin with.
2404    fn settle_questions(&mut self) {
2405        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2406            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2407        }
2408    }
2409
2410    /// The tail of the graph after a trustworthy tally: fold losers, review,
2411    /// gate, merge, and persist.
2412    fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2413        Box::pin(self.finish_after_tally_inner())
2414    }
2415
2416    async fn finish_after_tally_inner(&mut self) -> Result<()> {
2417        // Before the losers are folded: their branches are what the graft
2418        // reads from.
2419        self.graft().await?;
2420        if self.resource_halt()? {
2421            return Ok(());
2422        }
2423        self.fold_losers().await?;
2424        // Before review starts, and again right before the gate: a run's
2425        // review rounds can themselves take long enough for the base to move
2426        // a second time, and the gate is the one node whose "green" gets
2427        // acted on.
2428        self.sync_to_base().await?;
2429        if self.hold_base_sync().await? || self.resource_halt()? {
2430            return Ok(());
2431        }
2432        if self.state.status == RunStatus::AlreadyInBase {
2433            return Ok(());
2434        }
2435        self.review_loop().await?;
2436        if self.resource_halt()? {
2437            return Ok(());
2438        }
2439        self.sync_to_base().await?;
2440        if self.hold_base_sync().await? || self.resource_halt()? {
2441            return Ok(());
2442        }
2443        if self.state.status == RunStatus::AlreadyInBase {
2444            return Ok(());
2445        }
2446        self.gate().await?;
2447        self.merge().await?;
2448        if self.resource_halt()? {
2449            return Ok(());
2450        }
2451        self.state.save()?;
2452        Ok(())
2453    }
2454
2455    // ---------------------------------------------------------------- prep
2456
2457    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2458    /// run with the step and its output in the event; it is never swallowed.
2459    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2460    /// resume starts over instead of finding candidates and missing seats.
2461    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2462        let result = match ensure_setup_config(&mut self.state, repo).await {
2463            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2464            Err(e) => Err(e),
2465        };
2466        if let Err(e) = result {
2467            git::worktree_remove_clean(repo, wt).await.ok();
2468            if in_prep {
2469                self.rollback_prep(repo).await;
2470            }
2471            self.state.status = RunStatus::Blocked;
2472            self.state.event(
2473                "setup",
2474                format!("worktree setup failed in {}: {e:#}", wt.display()),
2475            );
2476            self.state.save()?;
2477            return Err(e);
2478        }
2479        Ok(())
2480    }
2481
2482    /// Undo everything `prep` created, so a resume re-runs it whole.
2483    async fn rollback_prep(&mut self, repo: &Path) {
2484        let root = self.state.worktree_root();
2485        for c in std::mem::take(&mut self.state.candidates) {
2486            if matches!(
2487                git::worktree_remove_clean(repo, &c.worktree).await,
2488                Ok(true)
2489            ) {
2490                git::branch_delete(repo, &c.branch).await.ok();
2491            } else {
2492                self.state.candidates.push(c);
2493            }
2494        }
2495        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2496        for j in 1..=seats {
2497            remove_read_seat_worktree(&mut self.state, &root.join(format!("judge-{j}")))
2498                .await
2499                .ok();
2500        }
2501        for k in 1..=self.state.config.graph.advisors {
2502            remove_read_seat_worktree(&mut self.state, &root.join(format!("advisor-{k}")))
2503                .await
2504                .ok();
2505        }
2506    }
2507
2508    async fn prep(&mut self) -> Result<()> {
2509        if let Some(reason) = &self.state.prep_failure {
2510            bail!(
2511                "candidate preparation previously failed: {reason}; correct the source before retrying"
2512            );
2513        }
2514        if self
2515            .state
2516            .prep_complete
2517            .unwrap_or(!self.state.candidates.is_empty())
2518        {
2519            return Ok(());
2520        }
2521        let repo = self.state.repo.clone();
2522        // Includes rollback, config, refs, checkout creation and setup. No
2523        // repository mutation may precede the real shared lock acquisition.
2524        let _coordination = self.coordinate(&repo).await?;
2525        let base = self.state.base_commit.clone();
2526        let plan = self.state.sources.plan(&repo, &base).await?;
2527        self.rollback_prep(&repo).await;
2528        if !self.state.candidates.is_empty() {
2529            bail!(
2530                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2531            );
2532        }
2533        self.state.prep_complete = Some(false);
2534        self.state.status = RunStatus::Prep;
2535        self.state.save()?;
2536        let repo = self.state.repo.clone();
2537        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2538        let root = self.state.worktree_root();
2539        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2540
2541        // The hook is the write-time half of the blindness contract; the
2542        // presentation filter in `blind` is the half that cannot be bypassed.
2543        let hooks_dir = self.state.dir().join("hooks");
2544        if self.state.config.blind.commit_msg_hook {
2545            std::fs::create_dir_all(&hooks_dir)
2546                .with_context(|| format!("create {}", hooks_dir.display()))?;
2547            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2548            let path = hooks_dir.join("commit-msg");
2549            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2550            make_executable(&path)?;
2551            // Ref-counted rather than a plain idempotent set: with more than
2552            // one run able to be in flight in the same repository at once
2553            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2554            // true?" check cannot tell "another run of mine still needs
2555            // this" from "nobody does", and the run that happens to finish
2556            // first would disable the hook out from under a sibling still
2557            // relying on it.
2558            git::acquire_worktree_config(&repo).await?;
2559            self.state.enabled_worktree_config = true;
2560        }
2561
2562        // `[worktree] setup` hides its products per worktree, which needs the
2563        // same per-worktree config the hook does. Held until fold, like it.
2564        ensure_setup_config(&mut self.state, &repo).await?;
2565
2566        for (index, (spec, label)) in self
2567            .roles
2568            .implementers
2569            .clone()
2570            .into_iter()
2571            .zip(labels)
2572            .enumerate()
2573        {
2574            let branch = self.state.branch_for(label);
2575            let worktree = root.join(format!("cand-{label}"));
2576            // Persist intent before Git can leave a partially created tree.
2577            // Candidate registration follows setup, so it cannot inventory
2578            // crashes or refusals during creation itself.
2579            self.state.event(
2580                "prep-intent",
2581                serde_json::json!({
2582                    "branch": branch,
2583                    "worktree": worktree,
2584                    "start": start,
2585                    "picks": plan.picks,
2586                })
2587                .to_string(),
2588            );
2589            self.state.save()?;
2590            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2591            if self.state.config.blind.commit_msg_hook {
2592                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2593            }
2594            git::local_exclude(&worktree, "/.magi/").await?;
2595            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2596                // Nothing of this prep may survive: a resume must start over,
2597                // not find half the candidates and skip the rest.
2598                git::worktree_remove_clean(&repo, &worktree).await.ok();
2599                git::branch_delete(&repo, &branch).await.ok();
2600                self.rollback_prep(&repo).await;
2601                self.state.status = RunStatus::Blocked;
2602                self.state
2603                    .event("prep", format!("worktree setup failed: {e:#}"));
2604                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2605                self.state.save()?;
2606                return Err(e);
2607            }
2608            self.state.candidates.push(Candidate {
2609                index,
2610                label,
2611                agent: spec.id.clone(),
2612                branch,
2613                worktree,
2614                summary: String::new(),
2615                stat: String::new(),
2616                files: 0,
2617                commits: 0,
2618                empty: false,
2619                failed: None,
2620                verified_noop: None,
2621                duration_ms: 0,
2622                folded: false,
2623            });
2624            self.state.save()?;
2625            for pick in &plan.picks {
2626                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2627                {
2628                    self.state.status = RunStatus::Blocked;
2629                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2630                    self.state
2631                        .event("prep", format!("cannot apply referenced commit: {e}"));
2632                    self.state.save()?;
2633                    return Err(e);
2634                }
2635            }
2636        }
2637
2638        for j in 1..=self.roles.judges.len() {
2639            let wt = root.join(format!("judge-{j}"));
2640            if !wt.exists() {
2641                git::worktree_add_detached(&repo, &wt, &base).await?;
2642                self.setup_seat_worktree(&repo, &wt, true).await?;
2643            }
2644        }
2645
2646        // Disposable, detached checkouts for the design-deliberation stage's
2647        // advisor seats — the same shape as the judges' above, at the same
2648        // base commit, since advisors also only ever read. Sized off the
2649        // configured count directly rather than a resolved roster: unlike
2650        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2651        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2652        // `prep` has no `ResolvedRoles` field to read a count from here.
2653        if self.state.config.graph.advise {
2654            for k in 1..=self.state.config.graph.advisors {
2655                let wt = root.join(format!("advisor-{k}"));
2656                if !wt.exists() {
2657                    git::worktree_add_detached(&repo, &wt, &base).await?;
2658                    self.setup_seat_worktree(&repo, &wt, true).await?;
2659                }
2660            }
2661        }
2662
2663        // A judge cannot tell it is looking at its own patch — the seats keep
2664        // separate conversations — but a panel that shares agents with the
2665        // field is less independent than it looks, and that is worth saying out
2666        // loud once per run rather than leaving it in the config.
2667        let authors: Vec<&str> = self
2668            .roles
2669            .implementers
2670            .iter()
2671            .map(|a| a.id.as_str())
2672            .collect();
2673        let overlap: Vec<String> = self
2674            .roles
2675            .judges
2676            .iter()
2677            .enumerate()
2678            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2679            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2680            .collect();
2681        if !overlap.is_empty() {
2682            let note = format!(
2683                "{} also authored a candidate; blind, but the panel is less \
2684                 independent than {} distinct agents would be",
2685                overlap.join(", "),
2686                self.roles.judges.len()
2687            );
2688            self.state.event("prep", note);
2689        }
2690
2691        self.state.event(
2692            "prep",
2693            format!(
2694                "{} candidates, {} judges, base {} ({})",
2695                self.state.candidates.len(),
2696                self.roles.judges.len(),
2697                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2698                self.state.base_branch
2699            ),
2700        );
2701        self.state.prep_complete = Some(true);
2702        self.state.status = RunStatus::Implementing;
2703        self.state.save()?;
2704        Ok(())
2705    }
2706
2707    // -------------------------------------------------------------- advise
2708
2709    /// The design-deliberation stage: independent, read-only advisor seats
2710    /// each sketch a design before any implementer touches the repository,
2711    /// and (when at least one produced a usable proposal) a synthesis seat
2712    /// blends them into a brief `implement` carries in every candidate's
2713    /// prompt.
2714    ///
2715    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2716    /// advisors` is the proposal count. Everything here is best-effort and
2717    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2718    /// that cannot reach quota, or a synthesis seat that produced nothing
2719    /// usable all leave `implement` exactly as it was before this stage
2720    /// existed — the task instruction alone — rather than failing the whole
2721    /// competition over an enrichment stage. Every outcome is still recorded
2722    /// as an event, so a run that got nothing from this stage says why.
2723    ///
2724    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2725    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2726    /// resumed run whose stage failed would re-run it, and re-spend the
2727    /// agent calls, on every reentry before `implement`.
2728    ///
2729    /// Also skipped once any candidate shows implementation progress — the
2730    /// exact predicate `implement` itself uses to decide a candidate is no
2731    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2732    /// is not enough: a run created by an older binary that predates this
2733    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2734    /// an already-`Implementing`-or-later run under this build would
2735    /// otherwise walk straight back through `prep` (a no-op once candidates
2736    /// exist) into this node and spawn every advisor seat against worktrees
2737    /// `prep` never recreated — after implementation has already started,
2738    /// which is exactly the invariant this stage exists to guarantee.
2739    async fn advise(&mut self) -> Result<()> {
2740        let implement_untouched = self
2741            .state
2742            .candidates
2743            .iter()
2744            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2745        if !self.state.config.graph.advise || self.state.advise_attempted {
2746            return Ok(());
2747        }
2748        if !implement_untouched {
2749            self.state.event(
2750                "advise",
2751                "skipping the design-deliberation stage: at least one \
2752                 candidate already shows implementation progress, so this \
2753                 run is past the point the stage exists to run before"
2754                    .to_owned(),
2755            );
2756            self.state.advise_attempted = true;
2757            self.state.save()?;
2758            return Ok(());
2759        }
2760        let run_id = self.state.id.clone();
2761        let prompts = self.state.config.prompts.clone();
2762        let instruction = self.state.instruction.clone();
2763        let language = self.state.config.graph.language.clone();
2764        let root = self.state.worktree_root();
2765        let n = self.state.config.graph.advisors;
2766        let where_recorded = self.state.dir().join("run.json");
2767
2768        let seats = match self.state.config.advisors() {
2769            Ok(seats) if !seats.is_empty() => seats,
2770            Ok(_) => {
2771                self.state.event(
2772                    "advise",
2773                    format!(
2774                        "[graph] advisors is 0; skipping the design-deliberation \
2775                         stage and continuing without a synthesis brief (see {})",
2776                        where_recorded.display()
2777                    ),
2778                );
2779                self.state.advise_attempted = true;
2780                self.state.save()?;
2781                return Ok(());
2782            }
2783            Err(e) => {
2784                self.state.event(
2785                    "advise",
2786                    format!(
2787                        "could not resolve advisor seats ({e:#}); continuing \
2788                         without a design-deliberation brief (see {})",
2789                        where_recorded.display()
2790                    ),
2791                );
2792                self.state.advise_attempted = true;
2793                self.state.save()?;
2794                return Ok(());
2795            }
2796        };
2797
2798        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2799        let artifacts = agent::artifacts_dir(&self.state.dir());
2800        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2801
2802        let mut jobs = Vec::new();
2803        for (i, spec) in seats.iter().cloned().enumerate() {
2804            let seat_key = format!("advisor-{}", i + 1);
2805            let seat = self.seat(&seat_key, &spec.id);
2806            jobs.push(SeatJob {
2807                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2808                spec,
2809                seat,
2810                cwd: worktrees[i % worktrees.len()].clone(),
2811                timeout,
2812                allow_write: false,
2813                sessions: false,
2814                artifacts: artifacts.clone(),
2815                stem: seat_key,
2816                handover: None,
2817            });
2818        }
2819
2820        self.state.event(
2821            "advise",
2822            format!(
2823                "{} advisor seat(s) sketching a design in parallel",
2824                jobs.len()
2825            ),
2826        );
2827        let mut quota_losses = Vec::new();
2828        let cache = self.state.config.cache_dir();
2829        let ctx = WaveCtx {
2830            carry_seats: false,
2831            pauses: Some((&self.pause, &self.interrupt)),
2832            run: &run_id,
2833            node: "advise",
2834            prompts: &prompts,
2835            cache: cache.as_deref(),
2836            round: None,
2837        };
2838        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2839        let results = ask_json_wave::<Proposal>(
2840            jobs,
2841            Arc::clone(&self.sem),
2842            self.state.config.graph.retries,
2843            &advisor_roster,
2844            &ctx,
2845            &mut quota_losses,
2846            &mut self.state,
2847            &|p: &Proposal| p.validate(),
2848        )
2849        .await;
2850        if self.state.resource_block.is_some() {
2851            return self.stop_for_resource();
2852        }
2853        self.state.quota.extend(quota_losses);
2854
2855        let mut records = Vec::with_capacity(results.len());
2856        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2857            let agent_id = seat.agent.clone();
2858            self.state.seats.insert(seat.key.clone(), seat);
2859            match res {
2860                Ok((proposal, out)) => {
2861                    self.state
2862                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2863                    records.push(advise::AdvisorRecord::proposed(
2864                        i + 1,
2865                        agent_id,
2866                        proposal,
2867                        out.duration_ms,
2868                    ));
2869                }
2870                Err(e) => {
2871                    self.state.event(
2872                        "advise",
2873                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2874                    );
2875                    records.push(advise::AdvisorRecord::failed(
2876                        i + 1,
2877                        agent_id,
2878                        e.to_string(),
2879                    ));
2880                }
2881            }
2882        }
2883
2884        let mut advice = advise::Advice {
2885            records,
2886            synthesis: None,
2887        };
2888        if advice.proposals().is_empty() {
2889            self.state.event(
2890                "advise",
2891                "no advisor produced a usable proposal; continuing without a \
2892                 synthesis brief"
2893                    .to_owned(),
2894            );
2895        } else {
2896            match self
2897                .synthesize_brief(
2898                    &advice,
2899                    &instruction,
2900                    &language,
2901                    &worktrees[0],
2902                    &artifacts,
2903                    &run_id,
2904                    &prompts,
2905                    cache.as_deref(),
2906                )
2907                .await
2908            {
2909                Ok(Some(text)) => {
2910                    self.state.event(
2911                        "advise",
2912                        "synthesized a design brief for the implementer".to_owned(),
2913                    );
2914                    advice.synthesis = Some(text);
2915                }
2916                Ok(None) => {
2917                    self.state.event(
2918                        "advise",
2919                        "the synthesis seat produced nothing usable; continuing \
2920                         without a design brief"
2921                            .to_owned(),
2922                    );
2923                }
2924                Err(e) => {
2925                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2926                        return Err(e);
2927                    }
2928                    self.state.event(
2929                        "advise",
2930                        format!("could not synthesize a design brief: {e:#}"),
2931                    );
2932                }
2933            }
2934        }
2935        advise::apply_reflection(&mut advice);
2936
2937        self.state.advice = Some(advice);
2938        self.state.advise_attempted = true;
2939        self.state.save()?;
2940        Ok(())
2941    }
2942
2943    /// The synthesis seat: reads every advisor's proposal and blends them
2944    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2945    /// out of [`Runner::advise`] only for readability — it is not called
2946    /// anywhere else.
2947    ///
2948    /// Picked the same way [`crate::talk`]'s standing conversation and
2949    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2950    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2951    /// order (a claude seat, else the first runnable agent in roster order)
2952    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2953    /// in [`crate::config`] for why a dedicated field exists here at all.
2954    #[allow(clippy::too_many_arguments)]
2955    async fn synthesize_brief(
2956        &mut self,
2957        advice: &advise::Advice,
2958        instruction: &str,
2959        language: &str,
2960        cwd: &Path,
2961        artifacts: &Path,
2962        run_id: &str,
2963        prompts: &Prompts,
2964        cache: Option<&Path>,
2965    ) -> Result<Option<String>> {
2966        let chain = agent::pick_chain(
2967            &self.state.config.agents,
2968            self.state.config.roles.synthesizer.as_ref(),
2969            &agent::installed,
2970            "synthesizer",
2971        )?;
2972        let proposals = advice.proposals();
2973        let mut prompt = prompt::with_overlay(
2974            prompt::synthesize_brief(instruction, &proposals, language),
2975            prompts.overlay("advise"),
2976        );
2977        if cache.is_some() {
2978            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2979            // below — see `prompt::build_cache_note`'s doc for why telling a
2980            // read-only seat to build through the shared cache is exactly how
2981            // a sandbox's write refusal gets misread as a defect.
2982            prompt.push('\n');
2983            prompt.push_str(&prompt::build_cache_note("advise", false));
2984        }
2985        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2986        // Each id is tried once, in order; a quota hit, error or unusable
2987        // answer moves to the next. The seat is single-turn (`sessions:
2988        // false`) and the prompt is the whole context, so a fallback agent
2989        // needs nothing carried over.
2990        let mut last = None;
2991        for (n, spec) in chain.iter().enumerate() {
2992            if n > 0 {
2993                self.state
2994                    .event("advise", format!("synthesis falling back to {}", spec.id));
2995            }
2996            let mut seat = self.seat("advise-synthesis", &spec.id);
2997            let wait = self.repository_wait();
2998            let outcome = wait
2999                .scope(agent::invoke(
3000                    spec,
3001                    &mut seat,
3002                    &Invocation {
3003                        coordination: self.state.config.coordination.as_ref(),
3004                        fetched_base: None,
3005                        cwd,
3006                        prompt: &prompt,
3007                        timeout,
3008                        allow_write: false,
3009                        sessions: false,
3010                        artifacts,
3011                        stem: &if n == 0 {
3012                            "advise-synthesis".to_owned()
3013                        } else {
3014                            format!("advise-synthesis-{}", spec.id)
3015                        },
3016                        run: run_id,
3017                        node: "advise",
3018                        cache_dir: None,
3019                        attachments: &[],
3020                        writable: &[],
3021                    },
3022                ))
3023                .await;
3024            if outcome.is_ok() {
3025                self.state.seats.insert(seat.key.clone(), seat);
3026            }
3027            let advance = agent::chain_advances(&outcome);
3028            last = Some(outcome);
3029            if !advance {
3030                break;
3031            }
3032        }
3033        // Exhausted: the last attempt's result is what a single failed seat
3034        // would have produced.
3035        let out = last.expect("a chain holds at least one agent")?;
3036        if !out.usable() {
3037            return Ok(None);
3038        }
3039        let text =
3040            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3041        Ok((!text.trim().is_empty()).then_some(text))
3042    }
3043
3044    // ----------------------------------------------------------- implement
3045
3046    async fn implement(&mut self) -> Result<()> {
3047        self.persist_candidates().await?;
3048        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3049        // agent files with `magi task add` name the run that paid for it. The
3050        // prompt overlay is cloned alongside it because the waves borrow it
3051        // while `self` is mutably borrowed by the node's own bookkeeping.
3052        let run_id = self.state.id.clone();
3053        let prompts = self.state.config.prompts.clone();
3054        let todo: Vec<usize> = self
3055            .state
3056            .candidates
3057            .iter()
3058            .enumerate()
3059            .filter(|(_, c)| {
3060                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3061                    && c.failed.is_none()
3062                    && !c.empty
3063            })
3064            .map(|(i, _)| i)
3065            .collect();
3066        if todo.is_empty() {
3067            return self.after_implement();
3068        }
3069        self.state.status = RunStatus::Implementing;
3070
3071        let language = self.state.config.graph.language.clone();
3072        let github_language = self.state.config.graph.github_language.clone();
3073        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3074        let sessions = self.state.config.graph.sessions;
3075        let artifacts = agent::artifacts_dir(&self.state.dir());
3076        // The design-deliberation stage's blended brief, when `advise` found
3077        // one — carried into every implementer's prompt the same way
3078        // regardless of which candidate it is.
3079        let brief = self
3080            .state
3081            .advice
3082            .as_ref()
3083            .and_then(|a| a.synthesis.as_deref())
3084            .map(str::to_owned);
3085        let attachments = self.state.attachments.clone();
3086
3087        let mut jobs = Vec::new();
3088        for &i in &todo {
3089            let (index, label, worktree) = {
3090                let c = &self.state.candidates[i];
3091                (c.index, c.label, c.worktree.clone())
3092            };
3093            let spec = self.roles.implementers[index].clone();
3094            let seat_key = format!("impl-{label}");
3095            let seat = self.seat(&seat_key, &spec.id);
3096            let instruction = seeded_instruction(&self.state);
3097            jobs.push(SeatJob {
3098                spec,
3099                seat,
3100                prompt: prompt::implement_in(
3101                    &instruction,
3102                    &worktree.to_string_lossy(),
3103                    &language,
3104                    &github_language,
3105                    brief.as_deref(),
3106                    &attachments,
3107                ),
3108                cwd: worktree,
3109                timeout,
3110                allow_write: true,
3111                sessions,
3112                artifacts: artifacts.clone(),
3113                stem: format!("impl-{label}"),
3114                handover: None,
3115            });
3116        }
3117
3118        self.state.event(
3119            "implement",
3120            format!("{} candidates in parallel", jobs.len()),
3121        );
3122        // Kept so a seat whose CLI hung up can be asked again from the same
3123        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
3124        // can update a seat's own entry once a fallback agent takes it over —
3125        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
3126        // whichever agent actually answered, not the one that quota'd out.
3127        let mut sent = jobs.clone();
3128        let cache = self.state.config.cache_dir();
3129        let ctx = WaveCtx {
3130            carry_seats: false,
3131            pauses: Some((&self.pause, &self.interrupt)),
3132            run: &run_id,
3133            node: "implement",
3134            prompts: &prompts,
3135            cache: cache.as_deref(),
3136            round: None,
3137        };
3138        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3139        if !wave_blocked(&results) {
3140            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3141                .await;
3142            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3143                .await?;
3144            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3145                .await;
3146        }
3147        // Do not finalize failed candidates when the entire wave needs machine
3148        // recovery: leaving them pending resumes this node with its work intact.
3149        if !results.is_empty()
3150            && results
3151                .iter()
3152                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3153        {
3154            let losses: Vec<_> = results
3155                .iter()
3156                .map(|(_, seat, out)| {
3157                    let AgentOutcome::Quota(o) = out else {
3158                        unreachable!()
3159                    };
3160                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3161                })
3162                .collect();
3163            self.state.quota.extend(losses.clone());
3164            for (_, seat, _) in results {
3165                self.state.seats.insert(seat.key.clone(), seat);
3166            }
3167            for &i in &todo {
3168                let label = self.state.candidates[i].label;
3169                self.state.interrupted_implementations.insert(label);
3170                self.state
3171                    .candidate_preservation
3172                    .insert(label, "machine outage awaiting persistence".into());
3173            }
3174            // Preserve partial edits too, while keeping every interrupted seat
3175            // pending even when its tree already contains an inherited patch.
3176            block_machine_outage(&mut self.state, "implement", &losses);
3177            self.state.save()?;
3178            self.persist_candidates().await?;
3179            for &i in &todo {
3180                self.state
3181                    .interrupted_implementations
3182                    .insert(self.state.candidates[i].label);
3183            }
3184            return self.stop_for_resource();
3185        }
3186        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3187
3188        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3189            && results
3190                .iter()
3191                .any(|(_, _, out)| FailClass::of(out).is_some());
3192        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3193            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3194                // Executed turns interrupted by disk pressure already carry
3195                // a preservation barrier from wave. Admission-only waits do
3196                // not. Keep the interrupted seat's session for recovery too.
3197                if self
3198                    .state
3199                    .candidate_preservation
3200                    .contains_key(&self.state.candidates[i].label)
3201                {
3202                    self.state.seats.insert(seat.key.clone(), seat);
3203                }
3204                continue;
3205            }
3206            let seat_key = seat.key.clone();
3207            // A quota fallback (`resume_seat_handovers`) may have handed this
3208            // seat to a different agent than the one `prep` recorded on the
3209            // candidate; the stats tables and any later fixer-defaults-to-
3210            // winner's-author lookup must credit whoever actually answered —
3211            // unless every fallback also quota'd out, in which case nobody
3212            // actually answered and crediting the last agent tried would
3213            // erase every earlier agent's own quota loss from the stats
3214            // tables instead of just this one seat's.
3215            let agent = seat.agent.clone();
3216            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3217            self.state.seats.insert(seat.key.clone(), seat);
3218            let label = self.state.candidates[i].label;
3219            self.state.interrupted_implementations.remove(&label);
3220
3221            let (summary, duration, failed, verified_claim) = match out {
3222                AgentOutcome::Ok(o) => {
3223                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3224                    let failed = (!o.usable()).then(|| {
3225                        if o.timed_out {
3226                            "agent timed out".to_owned()
3227                        } else {
3228                            format!(
3229                                "agent exited with {}",
3230                                crate::notices::exit_status(o.exit_code)
3231                            )
3232                        }
3233                    });
3234                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3235                    (text, o.duration_ms, failed, verified_claim)
3236                }
3237                // Left un-resumed by `resume_undelivered` (a dirty tree
3238                // already rescues the work, or there was no session left to
3239                // resume into) — reported like the ordinary failure it is,
3240                // never as if `o.text` (the CLI's raw error JSON) were an
3241                // answer.
3242                AgentOutcome::Dropped(o) => {
3243                    let why = o
3244                        .dropped
3245                        .as_ref()
3246                        .map(|d| d.why.as_str())
3247                        .unwrap_or("the CLI ended the stream without delivering its answer");
3248                    (
3249                        String::new(),
3250                        o.duration_ms,
3251                        Some(format!("the CLI dropped the stream ({why})")),
3252                        None,
3253                    )
3254                }
3255                AgentOutcome::Quota(o) => {
3256                    self.state.quota.push(QuotaLoss::from_report(
3257                        seat_key,
3258                        "implement",
3259                        o.quota.as_ref(),
3260                    ));
3261                    (
3262                        String::new(),
3263                        o.duration_ms,
3264                        Some("rate limited (quota); produced no change".to_owned()),
3265                        None,
3266                    )
3267                }
3268                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3269                    (String::new(), 0, Some(e), None)
3270                }
3271            };
3272
3273            // Save every completed turn before any fallible Git operation. A failed
3274            // rescue must not erase the reply or prevent collection of other seats.
3275            let c = &mut self.state.candidates[i];
3276            if !exhausted_the_fallback_chain {
3277                c.agent = agent;
3278            }
3279            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3280            c.duration_ms = duration;
3281            c.failed = failed;
3282            c.verified_noop = verified_claim;
3283            self.state
3284                .candidate_preservation
3285                .insert(label, "awaiting persistence".to_owned());
3286        }
3287        self.state.save()?;
3288        self.persist_candidates().await?;
3289
3290        if resource_blocked {
3291            return self.stop_for_resource();
3292        }
3293        if selected_failed {
3294            self.state.status = RunStatus::Blocked;
3295            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3296            self.state.save()?;
3297            bail!(
3298                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3299            );
3300        }
3301        self.after_implement()
3302    }
3303
3304    /// Retry only persistence after a completed implementation turn.
3305    async fn persist_candidates(&mut self) -> Result<()> {
3306        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3307        for label in labels {
3308            let i = self
3309                .state
3310                .candidates
3311                .iter()
3312                .position(|c| c.label == label)
3313                .context("pending candidate is missing")?;
3314            let c = self.state.candidates[i].clone();
3315            let interrupted = self.state.interrupted_implementations.contains(&label);
3316            let result = async {
3317                let wait = crate::repo_lock::WaitPolicy::new(
3318                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3319                    || false,
3320                );
3321                let _coordination =
3322                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3323                // Best effort evidence requires no index writes. A successful snapshot
3324                // never authorizes removal of a tree whose rescue failed.
3325                if let Err(e) = git::snapshot_uncommitted(
3326                    &c.worktree,
3327                    &self
3328                        .state
3329                        .dir()
3330                        .join("artifacts")
3331                        .join(format!("candidate-{label}-recovery")),
3332                )
3333                .await
3334                {
3335                    self.state.event(
3336                        "preserve",
3337                        format!("candidate {label} snapshot incomplete: {e:#}"),
3338                    );
3339                }
3340                let rescued = git::rescue_commit(
3341                    &c.worktree,
3342                    &format!("magi: candidate {label} (uncommitted work)"),
3343                )
3344                .await?;
3345                self.state.note_withheld("implement", &rescued.withheld);
3346                let base = &self.state.base_commit;
3347                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3348                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3349                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3350                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3351                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3352                if c.empty && commits > 0 && !patch.trim().is_empty()
3353                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3354                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3355                        || self.state.tally.is_some())
3356                {
3357                    // Keep the old classification and pending marker so every
3358                    // retry stops too. Retain decisions as recovery evidence;
3359                    // only a supported recovery operation may replace them.
3360                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3361                }
3362                let c = &mut self.state.candidates[i];
3363                c.commits = commits;
3364                c.stat = stat;
3365                c.files = files;
3366                let no_changes = commits == 0 || patch.trim().is_empty();
3367                c.empty = no_changes && !interrupted;
3368                if interrupted && no_changes {
3369                    c.failed = None;
3370                    c.verified_noop = None;
3371                }
3372                if !c.empty {
3373                    c.failed = None;
3374                    c.verified_noop = None;
3375                }
3376                self.state.event(
3377                    "preserve",
3378                    format!("candidate {label}: {files} files, {commits} commits"),
3379                );
3380                Ok::<_, anyhow::Error>(())
3381            }
3382            .await;
3383            match result {
3384                Ok(()) => {
3385                    self.state.candidate_preservation.remove(&label);
3386                    if !interrupted || self.state.candidates[i].files > 0 {
3387                        self.state.interrupted_implementations.remove(&label);
3388                    }
3389                }
3390                Err(e) => {
3391                    let reason = format!("candidate {label} preservation failed: {e:#}");
3392                    self.state.event("preserve", &reason);
3393                    self.state.candidate_preservation.insert(label, reason);
3394                }
3395            }
3396            if self.state.candidate_preservation.is_empty()
3397                && self.state.status == RunStatus::Blocked
3398                && self.state.candidates.iter().all(|c| c.failed.is_none())
3399            {
3400                // Persist the recovered phase with the cleared pending marker;
3401                // a crash between saves must not turn this into a selected-seat
3402                // failure hold on the next resume.
3403                self.state.status = RunStatus::Implementing;
3404                self.state.event(
3405                    "preserve",
3406                    "candidate persistence recovered; implementation can continue",
3407                );
3408            }
3409            self.state.save()?;
3410        }
3411        if !self.state.candidate_preservation.is_empty() {
3412            self.state.status = RunStatus::Blocked;
3413            self.state.save()?;
3414            bail!(
3415                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3416            );
3417        }
3418        Ok(())
3419    }
3420
3421    /// Ask again, once, for work a CLI did and then failed to hand over.
3422    ///
3423    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3424    /// status with an empty response and a usage report showing output tokens,
3425    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3426    /// seven minutes and 14,267 output tokens that arrived as an empty
3427    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3428    ///
3429    /// Two conditions, and both matter:
3430    ///
3431    /// - **Only when the tree is untouched.** Often the agent has already
3432    ///   written its files and only the closing message was lost; the rescue
3433    ///   commit below picks that up and there is nothing to ask for. Re-asking
3434    ///   then would pay for a second implementation of work already on disk.
3435    /// - **Once.** A CLI that drops one stream can drop the next, and this
3436    ///   node is the most expensive in the graph.
3437    ///
3438    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3439    /// dropped reply still carried its `conversation_id`, so the seat is asked
3440    /// to finish what it was doing rather than sent the whole task again. It
3441    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3442    /// node's - for the same reason a re-ranked judge does: restating finished
3443    /// work is not the work.
3444    ///
3445    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3446    /// same way until it resets, while an abandoned conversation is still
3447    /// there to be picked up.
3448    async fn resume_undelivered(
3449        &mut self,
3450        results: &mut [(usize, SeatState, AgentOutcome)],
3451        sent: &[SeatJob],
3452        prompts: &Prompts,
3453        run_id: &str,
3454    ) {
3455        for (wi, seat, out) in results.iter_mut() {
3456            let Some(dropped) = (match &*out {
3457                AgentOutcome::Dropped(o) => o.dropped.clone(),
3458                _ => None,
3459            }) else {
3460                continue;
3461            };
3462            let Some(job) = sent.get(*wi) else { continue };
3463            // Already on disk? Then only the closing message was lost.
3464            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3465                self.state.event(
3466                    "implement",
3467                    format!(
3468                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3469                         work is in the tree",
3470                        seat.key, dropped.output_tokens, dropped.why
3471                    ),
3472                );
3473                continue;
3474            }
3475            // The re-ask only makes sense as a resume: `resume_after_drop`
3476            // says nothing about the task, trusting the seat to still hold it.
3477            // Without a session to resume — sessions disabled, or this CLI's
3478            // drop shape happened not to carry a session id — that prompt
3479            // would open a brand-new conversation with no context at all,
3480            // which is worse than leaving this as the ordinary failure it
3481            // already is.
3482            if !has_context(&job.spec, seat, job.sessions) {
3483                self.state.event(
3484                    "implement",
3485                    format!(
3486                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3487                         is no session left to resume",
3488                        seat.key, dropped.output_tokens, dropped.why
3489                    ),
3490                );
3491                continue;
3492            }
3493            self.state.event(
3494                "implement",
3495                format!(
3496                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3497                     conversation",
3498                    seat.key, dropped.output_tokens, dropped.why
3499                ),
3500            );
3501            let mut retry = job.clone();
3502            retry.seat = seat.clone();
3503            retry.prompt = prompt::resume_after_drop(&dropped.why);
3504            retry.timeout = retry_budget(job.timeout, true);
3505            retry.stem = format!("{}-resume", job.stem);
3506            let cache = self.state.config.cache_dir();
3507            let ctx = WaveCtx {
3508                carry_seats: false,
3509                pauses: Some((&self.pause, &self.interrupt)),
3510                run: run_id,
3511                node: "implement",
3512                prompts,
3513                cache: cache.as_deref(),
3514                round: None,
3515            };
3516            let (resumed_seat, resumed) =
3517                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3518            *seat = resumed_seat;
3519            *out = resumed;
3520        }
3521    }
3522
3523    /// Fall an implement seat through to the next untried agent in the
3524    /// implementer roster when it lost to quota — or, since the handover was
3525    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3526    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3527    /// quota path itself being unchanged — instead of leaving the
3528    /// seat's loss final the moment one agent's account runs dry.
3529    ///
3530    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3531    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3532    /// truncates to the single slot rotation picked, so a solo task whose one
3533    /// implementer hits quota mid-run used to have nothing else to try. This
3534    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3535    /// unrotated roster — which is the only place the *other* candidates in
3536    /// the machine's roster still exist once `implementers` has been cut down
3537    /// to size.
3538    ///
3539    /// Walks forward from just past the seat's own original position in the
3540    /// roster, never wrapping back to the front: a later candidate slot (say
3541    /// `beta`, the roster's second entry) must fall through to the *next*
3542    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3543    /// almost certainly a different candidate's own agent already — and once
3544    /// the roster's tail is exhausted there is nothing left to fall through
3545    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3546    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3547    /// let this retry that id forever. The loop keeps falling through until
3548    /// an attempt lands something other than `Quota` or the roster's tail
3549    /// runs out of untried ids, at which point the seat is left exactly as
3550    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3551    /// `QuotaLoss` recorded, the candidate failed/empty.
3552    ///
3553    /// `sent` is taken mutably and updated with the fallback agent's spec:
3554    /// `resume_unconfirmed_commands`, which runs after this and also reads
3555    /// `sent`, must see whichever agent actually ended up answering the seat
3556    /// — reading the stale, original spec there would check session
3557    /// eligibility against the wrong CLI and could hand a fallback agent's
3558    /// session id to the agent that just lost the seat to quota.
3559    ///
3560    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3561    /// — `self.seat` only reuses state when the agent id is unchanged, so
3562    /// handing it a different id already gets this for free. Reusing the old
3563    /// seat would resume a different CLI's session as if it were a
3564    /// continuation of this one.
3565    ///
3566    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3567    /// a quota loss cuts an agent off mid-turn, so anything already in the
3568    /// tree is unfinished work, not a completed candidate a re-ask would pay
3569    /// for twice. A dirty tree is rescued into a commit first (the same
3570    /// neutral-identity rescue `implement`'s own outcome loop gives every
3571    /// candidate) so the next agent starts clean.
3572    ///
3573    /// The new agent gets the implementer's full prompt and full
3574    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3575    /// it has no session and no context, and is implementing the task from
3576    /// nothing, unlike a resumed drop which is only restating work already
3577    /// done.
3578    ///
3579    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3580    /// `implement` event, never into `self.state.quota` — that is what
3581    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3582    /// go unspent, and a seat that ultimately recovered on its second or
3583    /// third agent is not the stalled panel that check exists to catch. Only
3584    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3585    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3586    /// outcome loop already has — this helper never pushes to it itself.
3587    async fn resume_seat_handovers(
3588        &mut self,
3589        results: &mut [(usize, SeatState, AgentOutcome)],
3590        sent: &mut [SeatJob],
3591        prompts: &Prompts,
3592        run_id: &str,
3593    ) -> Result<()> {
3594        let instruction = seeded_instruction(&self.state);
3595        let language = self.state.config.graph.language.clone();
3596        let github_language = self.state.config.graph.github_language.clone();
3597        let brief = self
3598            .state
3599            .advice
3600            .as_ref()
3601            .and_then(|a| a.synthesis.as_deref())
3602            .map(str::to_owned);
3603        let attachments = self.state.attachments.clone();
3604        for (wi, seat, out) in results.iter_mut() {
3605            // Who holds the other candidate seats of this wave right now
3606            // (earlier handovers already written back to `sent`).
3607            let others: BTreeSet<String> = sent
3608                .iter()
3609                .enumerate()
3610                .filter(|(j, _)| j != wi)
3611                .map(|(_, j)| j.spec.id.clone())
3612                .collect();
3613            let Some(job) = sent.get_mut(*wi) else {
3614                continue;
3615            };
3616            // Where the seat's own original agent sits in the roster — the
3617            // fallback walk starts just past here, never at the front, so a
3618            // later candidate slot's quota loss does not fall back onto an
3619            // earlier slot's own agent.
3620            let start = self
3621                .roles
3622                .implementer_roster
3623                .iter()
3624                .position(|s| s.id == job.spec.id)
3625                .unwrap_or(0);
3626            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3627            let mut fallback_attempt = 0usize;
3628            let mut prev: Option<FailClass> = None;
3629            while let Some(cur) = FailClass::of(&*out) {
3630                if !should_hand_over(prev.as_ref(), &cur) {
3631                    break;
3632                }
3633                let Some(next) =
3634                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3635                        .cloned()
3636                else {
3637                    break;
3638                };
3639                tried.insert(next.id.clone());
3640                fallback_attempt += 1;
3641
3642                let coordination = self.coordinate(&job.cwd).await?;
3643                if let Ok(r) = git::rescue_commit(
3644                    &job.cwd,
3645                    &format!(
3646                        "magi: candidate {} (uncommitted work before {} fallback)",
3647                        seat.key,
3648                        if cur == FailClass::Quota {
3649                            "quota"
3650                        } else {
3651                            "handover"
3652                        }
3653                    ),
3654                )
3655                .await
3656                {
3657                    self.state.note_withheld("implement", &r.withheld);
3658                }
3659                drop(coordination);
3660
3661                record_handover(
3662                    &mut self.state,
3663                    "implement",
3664                    &seat.key,
3665                    &seat.agent,
3666                    &next.id,
3667                    &cur,
3668                    &fail_reason(&*out),
3669                );
3670                prev = Some(cur.clone());
3671
3672                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3673                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3674                // Kept in sync on `sent` itself, not just the local retry: a
3675                // later helper (`resume_unconfirmed_commands`) reads `sent`
3676                // after this one returns and must see whichever agent is now
3677                // occupying the seat, not the one that just quota'd out —
3678                // otherwise it would judge session/continuation eligibility
3679                // by the wrong CLI and could resend a fallback's session id
3680                // to the agent that lost it the seat in the first place.
3681                job.spec = next.clone();
3682                let mut retry = job.clone();
3683                retry.seat = new_seat;
3684                retry.prompt = prompt::implement_in(
3685                    &instruction,
3686                    &job.cwd.to_string_lossy(),
3687                    &language,
3688                    &github_language,
3689                    brief.as_deref(),
3690                    &attachments,
3691                );
3692                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3693                let cache = self.state.config.cache_dir();
3694                let ctx = WaveCtx {
3695                    carry_seats: false,
3696                    pauses: Some((&self.pause, &self.interrupt)),
3697                    run: run_id,
3698                    node: "implement",
3699                    prompts,
3700                    cache: cache.as_deref(),
3701                    round: None,
3702                };
3703                let (fallback_seat, fallback_out) = run_one(
3704                    retry,
3705                    Arc::clone(&self.sem),
3706                    &ctx,
3707                    &mut self.state,
3708                    fallback_attempt,
3709                )
3710                .await;
3711                *seat = fallback_seat;
3712                *out = fallback_out;
3713            }
3714        }
3715        Ok(())
3716    }
3717
3718    /// Ask an implement seat's own CLI to confirm what it started, once, when
3719    /// its reply reported a command whose completion status it never
3720    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3721    /// that does and does not mean.
3722    ///
3723    /// The completion contract this task asks for, extended to `implement`
3724    /// with the same signal `continue_fix_report` reads for the fixer,
3725    /// rather than a keyword search over the reply or a hard requirement on
3726    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3727    /// a candidate's CLI turn ended cleanly while a test run it had started
3728    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3729    /// named in it at all is untouched by this: `commands` is empty, so
3730    /// there is nothing to be unconfirmed.
3731    ///
3732    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3733    /// this is not about recovering edits that might already be on disk, it
3734    /// is about a result the seat itself never vouched for, which resuming
3735    /// asks for regardless of what the tree already holds. Bounded to one
3736    /// attempt for the same reason `resume_undelivered` is — this is the
3737    /// most expensive node in the graph — and a seat that still cannot
3738    /// confirm on that attempt is left as whatever its (possibly still
3739    /// unconfirmed) reply says; this does not invent a new "failed" reason
3740    /// for a candidate that otherwise produced a real, committed change.
3741    async fn resume_unconfirmed_commands(
3742        &mut self,
3743        results: &mut [(usize, SeatState, AgentOutcome)],
3744        sent: &[SeatJob],
3745        prompts: &Prompts,
3746        run_id: &str,
3747    ) {
3748        for (wi, seat, out) in results.iter_mut() {
3749            let AgentOutcome::Ok(o) = &*out else {
3750                continue;
3751            };
3752            if !has_unconfirmed_command(&o.commands) {
3753                continue;
3754            }
3755            let Some(job) = sent.get(*wi) else { continue };
3756            if !has_context(&job.spec, seat, job.sessions) {
3757                self.state.event(
3758                    "implement",
3759                    format!(
3760                        "{}: the reply named a command whose own CLI never confirmed the exit \
3761                         status of, but there is no session left to resume",
3762                        seat.key
3763                    ),
3764                );
3765                continue;
3766            }
3767            self.state.event(
3768                "implement",
3769                format!(
3770                    "{}: the reply named a command whose own CLI never confirmed the exit \
3771                     status of; resuming the conversation",
3772                    seat.key
3773                ),
3774            );
3775            let mut retry = job.clone();
3776            retry.seat = seat.clone();
3777            retry.prompt = prompt::resume_incomplete(
3778                "a command in your last reply had no confirmed exit status",
3779            );
3780            retry.timeout = retry_budget(job.timeout, true);
3781            retry.stem = format!("{}-confirm", job.stem);
3782            let cache = self.state.config.cache_dir();
3783            let ctx = WaveCtx {
3784                carry_seats: false,
3785                pauses: Some((&self.pause, &self.interrupt)),
3786                run: run_id,
3787                node: "implement",
3788                prompts,
3789                cache: cache.as_deref(),
3790                round: None,
3791            };
3792            let (resumed_seat, resumed) =
3793                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3794            *seat = resumed_seat;
3795            *out = resumed;
3796        }
3797    }
3798
3799    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3800    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3801    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3802    /// that motivated this.
3803    ///
3804    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3805    /// own nudge loop already covers for judge/review/vote seats, and not a
3806    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3807    /// implement seats: here the CLI turn genuinely finished while the node's
3808    /// own work — the fixer's account of what it did — had not. Gated purely
3809    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3810    /// reply, never on any wording in it, so a fixer whose valid, first-try
3811    /// `FixReport` happens to mention having waited on a background test is
3812    /// never resumed — the `Ok(report)` branch at the call site returns
3813    /// before this is ever invoked.
3814    ///
3815    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3816    /// attempt ([`retry_budget`]), nothing attempted once the session is
3817    /// gone, and a quota hit ends the loop immediately rather than retrying a
3818    /// rate limit that fails the same way again.
3819    async fn continue_fix_report(
3820        &mut self,
3821        mut seat: SeatState,
3822        parse_err: String,
3823        job: &SeatJob,
3824        prompts: &Prompts,
3825        run_id: &str,
3826        round: usize,
3827    ) -> (
3828        SeatState,
3829        Option<FixReport>,
3830        Option<String>,
3831        ContinuationRecord,
3832    ) {
3833        let mut last_err = parse_err;
3834        let mut cumulative_wait_ms = 0u64;
3835        let mut attempts = 0usize;
3836        loop {
3837            if !has_context(&job.spec, &seat, job.sessions) {
3838                self.state.event(
3839                    "fix",
3840                    format!(
3841                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3842                         session left to resume into"
3843                    ),
3844                );
3845                let outcome = if attempts == 0 {
3846                    ContinuationOutcome::NoSession
3847                } else {
3848                    ContinuationOutcome::Exhausted
3849                };
3850                return (
3851                    seat,
3852                    None,
3853                    Some(format!("unparsable fix report: {last_err}")),
3854                    ContinuationRecord {
3855                        attempts,
3856                        cumulative_wait_ms,
3857                        outcome,
3858                    },
3859                );
3860            }
3861            if attempts >= MAX_FIX_CONTINUATIONS {
3862                self.state.event(
3863                    "fix",
3864                    format!(
3865                        "round {round}: fixer's reply still had no adoption report after \
3866                         {attempts} continuation(s) ({last_err}); giving up"
3867                    ),
3868                );
3869                return (
3870                    seat,
3871                    None,
3872                    Some(format!(
3873                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3874                    )),
3875                    ContinuationRecord {
3876                        attempts,
3877                        cumulative_wait_ms,
3878                        outcome: ContinuationOutcome::Exhausted,
3879                    },
3880                );
3881            }
3882            attempts += 1;
3883            self.state.event(
3884                "fix",
3885                format!(
3886                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3887                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3888                ),
3889            );
3890            let mut retry = job.clone();
3891            retry.seat = seat.clone();
3892            retry.prompt = prompt::resume_incomplete(&last_err);
3893            retry.timeout = retry_budget(job.timeout, true);
3894            retry.stem = format!("{}-continue{attempts}", job.stem);
3895            let cache = self.state.config.cache_dir();
3896            let ctx = WaveCtx {
3897                carry_seats: false,
3898                pauses: Some((&self.pause, &self.interrupt)),
3899                run: run_id,
3900                node: "fix",
3901                prompts,
3902                cache: cache.as_deref(),
3903                round: Some(round),
3904            };
3905            let (resumed_seat, resumed_out) = run_one(
3906                retry,
3907                Arc::clone(&self.sem),
3908                &ctx,
3909                &mut self.state,
3910                attempts,
3911            )
3912            .await;
3913            seat = resumed_seat;
3914            match resumed_out {
3915                AgentOutcome::Ok(o) => {
3916                    cumulative_wait_ms += o.duration_ms;
3917                    match verdict::extract_json::<FixReport>(&o.text) {
3918                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3919                            self.state.event(
3920                                "fix",
3921                                format!(
3922                                    "round {round}: fixer's adoption report recovered after \
3923                                     {attempts} continuation(s)"
3924                                ),
3925                            );
3926                            return (
3927                                seat,
3928                                Some(report),
3929                                None,
3930                                ContinuationRecord {
3931                                    attempts,
3932                                    cumulative_wait_ms,
3933                                    outcome: ContinuationOutcome::Resumed,
3934                                },
3935                            );
3936                        }
3937                        // The report parsed, but this same reply's own
3938                        // CommandEvidence — the identical record `state.jobs`
3939                        // renders — names a command whose CLI never
3940                        // confirmed an exit status. Read together, that is
3941                        // not a resolved answer: keep nudging rather than
3942                        // accept a report standing next to a command the
3943                        // seat's own CLI cannot vouch for.
3944                        Ok(_) => {
3945                            last_err = "the reply parsed, but it reported a command whose own CLI \
3946                                 never confirmed an exit status"
3947                                .to_owned();
3948                        }
3949                        Err(e) => last_err = e.to_string(),
3950                    }
3951                }
3952                AgentOutcome::Quota(o) => {
3953                    cumulative_wait_ms += o.duration_ms;
3954                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3955                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3956                    self.state.quota.push(loss);
3957                    self.state.event(
3958                        "fix",
3959                        format!(
3960                            "round {round}: continuation rate limited (quota); not retrying now"
3961                        ),
3962                    );
3963                    return (
3964                        seat,
3965                        None,
3966                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3967                        ContinuationRecord {
3968                            attempts,
3969                            cumulative_wait_ms,
3970                            outcome: ContinuationOutcome::QuotaLost,
3971                        },
3972                    );
3973                }
3974                AgentOutcome::Dropped(o) => {
3975                    cumulative_wait_ms += o.duration_ms;
3976                    let why = o
3977                        .dropped
3978                        .as_ref()
3979                        .map(|d| d.why.as_str())
3980                        .unwrap_or("the CLI ended the stream without delivering its answer");
3981                    last_err = format!("the CLI dropped the stream ({why})");
3982                }
3983                AgentOutcome::Failed(e) => last_err = e,
3984                // The continuation was never started; the report stays lost
3985                // and the caller records that, without any handover.
3986                AgentOutcome::ResourceBlocked(e) => {
3987                    return (
3988                        seat,
3989                        None,
3990                        Some(format!("the continuation was not started: {e}")),
3991                        ContinuationRecord {
3992                            attempts: attempts.saturating_sub(1),
3993                            cumulative_wait_ms,
3994                            outcome: ContinuationOutcome::NoSession,
3995                        },
3996                    );
3997                }
3998            }
3999        }
4000    }
4001
4002    fn after_implement(&mut self) -> Result<()> {
4003        if !self.state.candidate_preservation.is_empty() {
4004            bail!("candidate preservation incomplete; judging is unsafe");
4005        }
4006        // Scan every candidate patch once the set is complete.
4007        if self.state.leaks.is_empty() {
4008            let cfg = self.state.config.blind.clone();
4009            let mut leaks = Vec::new();
4010            for c in &self.state.candidates {
4011                let Some(patch) =
4012                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4013                else {
4014                    continue;
4015                };
4016                leaks.extend(blind::scan(
4017                    &format!("candidate {} patch", c.label),
4018                    &patch,
4019                    &cfg.vendor_tokens,
4020                ));
4021            }
4022            if !leaks.is_empty() {
4023                let summary = leaks
4024                    .iter()
4025                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4026                    .collect::<Vec<_>>()
4027                    .join(", ");
4028                match cfg.on_leak {
4029                    LeakPolicy::Fail => {
4030                        self.state.status = RunStatus::Failed;
4031                        self.state
4032                            .event("blind", format!("vendor text in a patch: {summary}"));
4033                        self.state.leaks = leaks;
4034                        self.state.save()?;
4035                        self.settle_questions();
4036                        bail!(
4037                            "blind.on_leak = \"fail\" and vendor text reached a \
4038                             judged patch: {summary}"
4039                        );
4040                    }
4041                    LeakPolicy::Redact => self.state.event(
4042                        "blind",
4043                        format!("redacting vendor text for judging: {summary}"),
4044                    ),
4045                    LeakPolicy::Warn => self.state.event(
4046                        "blind",
4047                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
4048                    ),
4049                }
4050                self.state.leaks = leaks;
4051            }
4052        }
4053
4054        if self.state.viable().is_empty() {
4055            if self.state.all_candidates_verified_noop() {
4056                // Every candidate agreed, with evidence the adoption guard
4057                // accepted, that nothing belongs in this worktree. That is
4058                // not the same fact as a candidate that simply failed to
4059                // write anything, and settling it as an ordinary `Failed`
4060                // (see `SCHEMA`'s doc for schema 10) is what let two of
4061                // task 391f's attempts burn a retry each re-discovering the
4062                // same already-landed fix. Terminal either way, so `judge`
4063                // must never run over an empty candidate set — unlike the
4064                // `Failed` branch below this returns `Ok`, not an error:
4065                // nothing here failed.
4066                self.state.status = RunStatus::VerifiedNoop;
4067                self.state.save()?;
4068                self.settle_questions();
4069                return Ok(());
4070            }
4071            self.state.status = RunStatus::Failed;
4072            self.state.save()?;
4073            self.settle_questions();
4074            bail!("no candidate produced a change; nothing to judge");
4075        }
4076        self.state.status = RunStatus::Judging;
4077        self.state.save()?;
4078        Ok(())
4079    }
4080
4081    // --------------------------------------------------------------- judge
4082
4083    async fn judge(&mut self) -> Result<()> {
4084        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4085        // agent files with `magi task add` name the run that paid for it. The
4086        // prompt overlay is cloned alongside it because the waves borrow it
4087        // while `self` is mutably borrowed by the node's own bookkeeping.
4088        let run_id = self.state.id.clone();
4089        let prompts = self.state.config.prompts.clone();
4090        if !self.state.judgements.is_empty() || self.state.judge_skipped {
4091            return Ok(());
4092        }
4093        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4094        if viable.len() == 1 {
4095            // Recorded so this is a one-time event: `judgements` stays empty
4096            // either way, which without this flag is indistinguishable from
4097            // "not yet judged" on the next reentry — and status is left
4098            // untouched, so a later node's conclusion (e.g. `Blocked` after
4099            // the review budget ran out) survives a resume instead of being
4100            // clobbered back to `Judging` by this node running again.
4101            self.state.judge_skipped = true;
4102            self.state.event(
4103                "judge",
4104                format!(
4105                    "only candidate {} produced a change; judging skipped",
4106                    viable[0].label
4107                ),
4108            );
4109            self.state.save()?;
4110            return Ok(());
4111        }
4112        self.state.status = RunStatus::Judging;
4113
4114        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4115        let language = self.state.config.graph.language.clone();
4116        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4117        let sessions = self.state.config.graph.sessions;
4118        let artifacts = agent::artifacts_dir(&self.state.dir());
4119        let root = self.state.worktree_root();
4120        let base_short = short(&self.state.base_commit);
4121
4122        let mut jobs = Vec::new();
4123        let mut orders = Vec::new();
4124        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4125            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4126            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4127            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4128            let seat_key = format!("judge-{}", j + 1);
4129            let seat = self.seat(&seat_key, &spec.id);
4130            jobs.push(SeatJob {
4131                prompt: prompt::judge(
4132                    &self.state.instruction,
4133                    &views,
4134                    self.roles.judges.len(),
4135                    &base_short,
4136                    &language,
4137                ),
4138                spec,
4139                seat,
4140                cwd: root.join(format!("judge-{}", j + 1)),
4141                timeout,
4142                allow_write: false,
4143                sessions,
4144                artifacts: artifacts.clone(),
4145                stem: format!("judge-{}", j + 1),
4146                handover: None,
4147            });
4148        }
4149
4150        self.state.event(
4151            "judge",
4152            format!(
4153                "{} judges ranking {} candidates blind",
4154                jobs.len(),
4155                viable.len()
4156            ),
4157        );
4158        let labels_for_check = labels.clone();
4159        let mut quota_losses = Vec::new();
4160        let cache = self.state.config.cache_dir();
4161        let ctx = WaveCtx {
4162            carry_seats: false,
4163            pauses: Some((&self.pause, &self.interrupt)),
4164            run: &run_id,
4165            node: "judge",
4166            prompts: &prompts,
4167            cache: cache.as_deref(),
4168            round: None,
4169        };
4170        let results = ask_json_wave::<Ranking>(
4171            jobs,
4172            Arc::clone(&self.sem),
4173            self.state.config.graph.retries,
4174            &self.roles.judge_roster,
4175            &ctx,
4176            &mut quota_losses,
4177            &mut self.state,
4178            &move |r: &Ranking| r.validate(&labels_for_check),
4179        )
4180        .await;
4181        if self.state.resource_block.is_some() {
4182            return self.stop_for_resource();
4183        }
4184        self.state.quota.extend(quota_losses);
4185
4186        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4187            let agent_id = seat.agent.clone();
4188            self.state.seats.insert(seat.key.clone(), seat);
4189            let mut record = Judgement {
4190                judge: j + 1,
4191                seat: format!("judge-{}", j + 1),
4192                agent: agent_id,
4193                ranking: Vec::new(),
4194                reasons: BTreeMap::new(),
4195                confidence: None,
4196                strengths: BTreeMap::new(),
4197                order: orders[j].clone(),
4198                failed: None,
4199                duration_ms: 0,
4200            };
4201            match res {
4202                Ok((ranking, out)) => {
4203                    record.ranking = ranking.normalized();
4204                    record.reasons = ranking.reasons;
4205                    record.strengths = ranking.strengths;
4206                    record.confidence = ranking.confidence;
4207                    record.duration_ms = out.duration_ms;
4208                    self.state.event(
4209                        "judge",
4210                        format!(
4211                            "judge {} ranked {}",
4212                            j + 1,
4213                            record.ranking.iter().collect::<String>()
4214                        ),
4215                    );
4216                }
4217                Err(e) => {
4218                    record.failed = Some(e.to_string());
4219                    self.state
4220                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4221                }
4222            }
4223            self.state.judgements.push(record);
4224            self.state.save()?;
4225        }
4226        Ok(())
4227    }
4228
4229    // ---------------------------------------------------------- deliberate
4230
4231    async fn deliberate(&mut self) -> Result<()> {
4232        if self.hold_base_sync().await? {
4233            return Ok(());
4234        }
4235        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4236        // agent files with `magi task add` name the run that paid for it. The
4237        // prompt overlay is cloned alongside it because the waves borrow it
4238        // while `self` is mutably borrowed by the node's own bookkeeping.
4239        let run_id = self.state.id.clone();
4240        let prompts = self.state.config.prompts.clone();
4241        let rounds = self.state.config.graph.deliberate_rounds;
4242        // Resume unfinished rounds after an outage, but never reopen a tally.
4243        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4244            return Ok(());
4245        }
4246        let tops: Vec<char> = self
4247            .state
4248            .judgements
4249            .iter()
4250            .filter_map(|j| j.ranking.first().copied())
4251            .collect();
4252        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4253            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4254                self.state.event(
4255                    "deliberate",
4256                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4257                );
4258            }
4259            self.state.status = RunStatus::Voting;
4260            self.state.save()?;
4261            return Ok(());
4262        }
4263
4264        self.state.status = RunStatus::Deliberating;
4265        self.state.event(
4266            "deliberate",
4267            format!(
4268                "split: first choices were {} — opening {rounds} round(s)",
4269                tops.iter().collect::<String>()
4270            ),
4271        );
4272
4273        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4274        let language = self.state.config.graph.language.clone();
4275        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4276        let sessions = self.state.config.graph.sessions;
4277        let artifacts = agent::artifacts_dir(&self.state.dir());
4278        let root = self.state.worktree_root();
4279        let base_short = short(&self.state.base_commit);
4280
4281        // Judges argue in sequence so that a turn can answer the one before it;
4282        // that is the difference between deliberation and three parallel
4283        // monologues.
4284        for round in self.state.deliberation.len() + 1..=rounds {
4285            let mut turns: Vec<DeliberationTurn> = Vec::new();
4286            let mut round_losses = Vec::new();
4287            let mut asked = 0;
4288            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4289                if self.state.judgements[j].failed.is_some() {
4290                    continue;
4291                }
4292                let seat_key = format!("judge-{}", j + 1);
4293                let spec = self.occupant(&seat_key, spec);
4294                let mut seat = self.seat(&seat_key, &spec.id);
4295                let transcript = self.transcript(&turns, j);
4296                let build = |context: Option<&str>| {
4297                    prompt::deliberate(
4298                        &self.state.instruction,
4299                        context,
4300                        &transcript,
4301                        round,
4302                        rounds,
4303                        &language,
4304                    )
4305                };
4306                let block = self.candidate_block(&viable, &base_short);
4307                let full = build(Some(&block));
4308                let text = if has_context(&spec, &seat, sessions) {
4309                    build(None)
4310                } else {
4311                    full.clone()
4312                };
4313                let job = SeatJob {
4314                    spec,
4315                    seat: seat.clone(),
4316                    prompt: text,
4317                    cwd: root.join(format!("judge-{}", j + 1)),
4318                    timeout,
4319                    allow_write: false,
4320                    sessions,
4321                    artifacts: artifacts.clone(),
4322                    stem: format!("delib-{round}-judge-{}", j + 1),
4323                    handover: Some(full),
4324                };
4325                let cache = self.state.config.cache_dir();
4326                let ctx = WaveCtx {
4327                    carry_seats: false,
4328                    pauses: Some((&self.pause, &self.interrupt)),
4329                    run: &run_id,
4330                    node: "deliberate",
4331                    prompts: &prompts,
4332                    cache: cache.as_deref(),
4333                    round: None,
4334                };
4335                // A turn is never nudged (`retries` 0): a failed seat is
4336                // handed to the next roster agent, which gets the full
4337                // context. An empty answer is a turn, not a failure.
4338                let mut losses = Vec::new();
4339                asked += 1;
4340                let mut results = ask_wave_with::<String>(
4341                    vec![job],
4342                    Arc::clone(&self.sem),
4343                    0,
4344                    &self.roles.judge_roster,
4345                    &ctx,
4346                    &mut losses,
4347                    &mut self.state,
4348                    &|text: &str| {
4349                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4350                    },
4351                )
4352                .await;
4353                if self.state.resource_block.is_some() {
4354                    return self.stop_for_resource();
4355                }
4356                round_losses.extend(losses.iter().cloned());
4357                self.state.quota.extend(losses);
4358                let (updated, res, _) = results.pop().expect("one job in, one result out");
4359                seat = updated;
4360                let agent_id = seat.agent.clone();
4361                self.state.seats.insert(seat.key.clone(), seat);
4362                let body = match res {
4363                    Ok((body, _)) => body,
4364                    // Skip the seat; a CLI's raw error JSON is never read as
4365                    // this judge's position.
4366                    Err(e) => {
4367                        self.state
4368                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4369                        continue;
4370                    }
4371                };
4372                let tentative = verdict::extract_json::<Position>(&body)
4373                    .ok()
4374                    .and_then(|p| p.tentative)
4375                    .and_then(|s| s.trim().chars().next())
4376                    .map(|c| c.to_ascii_uppercase());
4377                self.state.event(
4378                    "deliberate",
4379                    format!(
4380                        "round {round}: judge {} now favours {}",
4381                        j + 1,
4382                        tentative.map_or("—".to_owned(), |c| c.to_string())
4383                    ),
4384                );
4385                turns.push(DeliberationTurn {
4386                    judge: j + 1,
4387                    agent: agent_id,
4388                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4389                    tentative,
4390                });
4391            }
4392            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4393                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4394                return self.stop_for_resource();
4395            }
4396            self.state
4397                .deliberation
4398                .push(DeliberationRound { round, turns });
4399            self.state.save()?;
4400        }
4401
4402        self.state.status = RunStatus::Voting;
4403        self.state.save()?;
4404        Ok(())
4405    }
4406
4407    // ---------------------------------------------------------------- vote
4408
4409    async fn vote(&mut self) -> Result<()> {
4410        if self.hold_base_sync().await? {
4411            return Ok(());
4412        }
4413        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4414        // agent files with `magi task add` name the run that paid for it. The
4415        // prompt overlay is cloned alongside it because the waves borrow it
4416        // while `self` is mutably borrowed by the node's own bookkeeping.
4417        let run_id = self.state.id.clone();
4418        let prompts = self.state.config.prompts.clone();
4419        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4420            return Ok(());
4421        }
4422        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4423        if viable.len() == 1 {
4424            return Ok(());
4425        }
4426        self.state.status = RunStatus::Voting;
4427
4428        let language = self.state.config.graph.language.clone();
4429        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4430        let sessions = self.state.config.graph.sessions;
4431        let artifacts = agent::artifacts_dir(&self.state.dir());
4432        let root = self.state.worktree_root();
4433        let base_short = short(&self.state.base_commit);
4434        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4435
4436        let mut jobs = Vec::new();
4437        let mut seats_at = Vec::new();
4438        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4439            if self
4440                .state
4441                .judgements
4442                .get(j)
4443                .is_some_and(|r| r.failed.is_some())
4444            {
4445                continue;
4446            }
4447            let seat_key = format!("judge-{}", j + 1);
4448            let spec = self.occupant(&seat_key, spec);
4449            let seat = self.seat(&seat_key, &spec.id);
4450            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4451            let text = if has_context(&spec, &seat, sessions) {
4452                prompt::final_vote(&viable, &language)
4453            } else {
4454                full.clone()
4455            };
4456            jobs.push(SeatJob {
4457                spec,
4458                seat,
4459                prompt: text,
4460                cwd: root.join(format!("judge-{}", j + 1)),
4461                timeout,
4462                allow_write: false,
4463                sessions,
4464                artifacts: artifacts.clone(),
4465                stem: format!("vote-judge-{}", j + 1),
4466                handover: Some(full),
4467            });
4468            seats_at.push(j);
4469        }
4470
4471        self.state.event(
4472            "vote",
4473            format!(
4474                "collecting {} final votes one by one, privately",
4475                jobs.len()
4476            ),
4477        );
4478        let allowed = viable.clone();
4479        let mut quota_losses = Vec::new();
4480        let cache = self.state.config.cache_dir();
4481        let ctx = WaveCtx {
4482            carry_seats: false,
4483            pauses: Some((&self.pause, &self.interrupt)),
4484            run: &run_id,
4485            node: "vote",
4486            prompts: &prompts,
4487            cache: cache.as_deref(),
4488            round: None,
4489        };
4490        let results = ask_json_wave::<FinalVote>(
4491            jobs,
4492            Arc::clone(&self.sem),
4493            self.state.config.graph.retries,
4494            &self.roles.judge_roster,
4495            &ctx,
4496            &mut quota_losses,
4497            &mut self.state,
4498            &move |v: &FinalVote| match v.label() {
4499                Some(c) if allowed.contains(&c) => Ok(()),
4500                other => bail!("vote {other:?} is not one of {allowed:?}"),
4501            },
4502        )
4503        .await;
4504        if self.state.resource_block.is_some() {
4505            return self.stop_for_resource();
4506        }
4507        self.state.quota.extend(quota_losses);
4508
4509        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4510            let agent_id = seat.agent.clone();
4511            self.state.seats.insert(seat.key.clone(), seat);
4512            let initial = self
4513                .state
4514                .judgements
4515                .get(j)
4516                .and_then(|r| r.ranking.first().copied());
4517            let mut record = VoteRecord {
4518                judge: j + 1,
4519                agent: agent_id,
4520                vote: None,
4521                reason: String::new(),
4522                changed: false,
4523            };
4524            match res {
4525                Ok((v, _)) => {
4526                    record.vote = v.label();
4527                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4528                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4529                    self.state.event(
4530                        "vote",
4531                        format!(
4532                            "judge {} voted {}{}",
4533                            j + 1,
4534                            record.vote.unwrap_or('?'),
4535                            if record.changed { " (changed)" } else { "" }
4536                        ),
4537                    );
4538                }
4539                Err(e) => {
4540                    self.state
4541                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4542                }
4543            }
4544            self.state.votes.push(record);
4545            self.state.save()?;
4546        }
4547        Ok(())
4548    }
4549
4550    // --------------------------------------------------------------- tally
4551
4552    fn tally(&mut self) -> Result<()> {
4553        if self.state.tally.is_some() {
4554            return Ok(());
4555        }
4556        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4557        let tops: Vec<char> = self
4558            .state
4559            .judgements
4560            .iter()
4561            .filter_map(|j| j.ranking.first().copied())
4562            .collect();
4563        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4564
4565        // A judge whose private vote failed still counted once, in the initial
4566        // ranking; using it beats discarding a whole seat.
4567        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4568        let mut cast: Vec<char> = Vec::new();
4569        for (i, j) in self.state.judgements.iter().enumerate() {
4570            let vote = self
4571                .state
4572                .votes
4573                .iter()
4574                .find(|v| v.judge == i + 1)
4575                .and_then(|v| v.vote)
4576                .or_else(|| j.ranking.first().copied());
4577            if let Some(v) = vote {
4578                *first_choice.entry(v).or_insert(0) += 1;
4579                cast.push(v);
4580            }
4581        }
4582
4583        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4584        for j in &self.state.judgements {
4585            let n = j.ranking.len();
4586            for (pos, label) in j.ranking.iter().enumerate() {
4587                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4588            }
4589        }
4590
4591        let best = first_choice.values().copied().max().unwrap_or(0);
4592        let mut leaders: Vec<char> = first_choice
4593            .iter()
4594            .filter(|(_, v)| **v == best)
4595            .map(|(k, _)| *k)
4596            .collect();
4597        let mut tie_break = None;
4598        if leaders.len() > 1 {
4599            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4600            let borda_leaders: Vec<char> = leaders
4601                .iter()
4602                .copied()
4603                .filter(|l| borda[l] == top_borda)
4604                .collect();
4605            tie_break = Some(if borda_leaders.len() == 1 {
4606                format!(
4607                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4608                    leaders.len()
4609                )
4610            } else {
4611                format!(
4612                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4613                    leaders.len()
4614                )
4615            });
4616            leaders = borda_leaders;
4617            leaders.sort_unstable();
4618        }
4619        let winner = *leaders
4620            .first()
4621            .or(viable.first())
4622            .context("no candidate to declare a winner from")?;
4623
4624        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4625        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4626        let deliberated = !self.state.deliberation.is_empty();
4627
4628        // Whose verdict is this? A rate-limited seat is absent even if it
4629        // ranked before the limit hit, so presence is measured against the
4630        // recorded losses, not just "did a ranking ever appear".
4631        let quota_seats: std::collections::BTreeSet<&str> =
4632            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4633        let mut present = 0usize;
4634        for (i, j) in self.state.judgements.iter().enumerate() {
4635            if quota_seats.contains(j.seat.as_str()) {
4636                continue;
4637            }
4638            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4639            let voted = self
4640                .state
4641                .votes
4642                .iter()
4643                .any(|v| v.judge == i + 1 && v.vote.is_some());
4644            if ranked || voted {
4645                present += 1;
4646            }
4647        }
4648        // Strict majority of the configured panel. A bare majority is real
4649        // signal we can act on, while a minority verdict must never stand in
4650        // for a healthy one. A one-candidate run needs no panel at all, and
4651        // `judges` stays `0` rather than the roster size a panel that never
4652        // sat would otherwise be credited with.
4653        let needs_quorum = viable.len() > 1;
4654        let judges_total = if needs_quorum {
4655            self.roles.judges.len()
4656        } else {
4657            0
4658        };
4659        let quorum = if needs_quorum {
4660            judges_total / 2 + 1
4661        } else {
4662            0
4663        };
4664        let met_quorum = !needs_quorum || present >= quorum;
4665        let uncontested = (!needs_quorum).then(|| {
4666            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4667        });
4668
4669        self.state.event(
4670            "tally",
4671            match &uncontested {
4672                Some(reason) => format!("winner {winner} — {reason}"),
4673                None => format!(
4674                    "winner {winner} — votes {} | initial {} | {} changed | \
4675                     {present}/{judges_total} judges{}",
4676                    first_choice
4677                        .iter()
4678                        .map(|(k, v)| format!("{k}:{v}"))
4679                        .collect::<Vec<_>>()
4680                        .join(" "),
4681                    if unanimous_initial {
4682                        "unanimous"
4683                    } else {
4684                        "split"
4685                    },
4686                    changed_votes,
4687                    if met_quorum {
4688                        String::new()
4689                    } else {
4690                        format!(" — below quorum ({quorum} required)")
4691                    },
4692                ),
4693            },
4694        );
4695        if !met_quorum {
4696            self.state.event(
4697                "stall",
4698                format!(
4699                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4700                     the run stops here, resumable"
4701                ),
4702            );
4703        }
4704        self.state.tally = Some(Tally {
4705            first_choice,
4706            borda,
4707            winner,
4708            rankings: tops.len(),
4709            unanimous_initial,
4710            deliberated,
4711            changed_votes,
4712            unanimous_final,
4713            tie_break,
4714            judges: judges_total,
4715            present,
4716            quorum,
4717            met_quorum,
4718            uncontested,
4719        });
4720        self.state.status = if met_quorum {
4721            RunStatus::Reviewing
4722        } else {
4723            RunStatus::Stalled
4724        };
4725        self.state.save()?;
4726        Ok(())
4727    }
4728
4729    // ------------------------------------------------------------- recover
4730
4731    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4732    /// resumed toward completion once the transient cause clears.
4733    ///
4734    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4735    /// it toward the quorum, which is exactly the set of seats whose absence
4736    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4737    /// not depend on which node happened to hit the limit), or an ordinary
4738    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4739    /// seat is never disturbed.
4740    ///
4741    /// A seat that now answers with a usable ranking is "recovered": its
4742    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4743    /// `tally` counts it present again), and its vote re-collected. A seat that
4744    /// still fails keeps its loss and stays absent.
4745    ///
4746    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4747    /// to review/gate/merge), `false` when it is still below quorum (the run
4748    /// stays `Stalled`, still resumable for a later retry).
4749    #[allow(clippy::too_many_lines)]
4750    async fn recover_stall(&mut self) -> Result<bool> {
4751        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4752        // agent files with `magi task add` name the run that paid for it. The
4753        // prompt overlay is cloned alongside it because the waves borrow it
4754        // while `self` is mutably borrowed by the node's own bookkeeping.
4755        let run_id = self.state.id.clone();
4756        let prompts = self.state.config.prompts.clone();
4757        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4758        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4759        // is what keeps a non-quota collapse — or a quota loss recorded at the
4760        // deliberate node — from being a permanent dead-end on `--resume`.
4761        let quota_seats: BTreeSet<&str> =
4762            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4763        let absent: Vec<String> = self
4764            .state
4765            .judgements
4766            .iter()
4767            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4768            .map(|j| j.seat.clone())
4769            .collect();
4770        if absent.is_empty() {
4771            return Ok(false);
4772        }
4773        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4774        if viable.len() <= 1 {
4775            return Ok(false);
4776        }
4777        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4778        let language = self.state.config.graph.language.clone();
4779        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4780        let sessions = self.state.config.graph.sessions;
4781        let artifacts = agent::artifacts_dir(&self.state.dir());
4782        let root = self.state.worktree_root();
4783        let base_short = short(&self.state.base_commit);
4784        let candidates: Vec<Candidate> = viable.clone();
4785
4786        // Map each absent seat key to its 0-based position in `roles.judges`.
4787        let mut positions: Vec<usize> = absent
4788            .iter()
4789            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4790            .collect();
4791        if positions.is_empty() {
4792            return Ok(false);
4793        }
4794        positions.sort_unstable();
4795        positions.dedup();
4796
4797        // Re-rank the lost seats, one blind prompt each.
4798        let mut judge_jobs = Vec::new();
4799        for &j in &positions {
4800            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4801            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4802            let seat_key = format!("judge-{}", j + 1);
4803            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4804            let seat = self.seat(&seat_key, &spec.id);
4805            judge_jobs.push(SeatJob {
4806                spec,
4807                seat,
4808                prompt: prompt::judge(
4809                    &self.state.instruction,
4810                    &views,
4811                    self.roles.judges.len(),
4812                    &base_short,
4813                    &language,
4814                ),
4815                cwd: root.join(seat_key),
4816                timeout,
4817                allow_write: false,
4818                sessions,
4819                artifacts: artifacts.clone(),
4820                stem: format!("judge-{}-recover", j + 1),
4821                handover: None,
4822            });
4823        }
4824
4825        let labels_for_check = labels.clone();
4826        let mut judge_losses = Vec::new();
4827        let retries = self.state.config.graph.retries;
4828        let cache = self.state.config.cache_dir();
4829        let ctx = WaveCtx {
4830            carry_seats: false,
4831            pauses: Some((&self.pause, &self.interrupt)),
4832            run: &run_id,
4833            node: "judge",
4834            prompts: &prompts,
4835            cache: cache.as_deref(),
4836            round: None,
4837        };
4838        let results = ask_json_wave::<Ranking>(
4839            judge_jobs,
4840            Arc::clone(&self.sem),
4841            retries,
4842            &self.roles.judge_roster,
4843            &ctx,
4844            &mut judge_losses,
4845            &mut self.state,
4846            &move |r: &Ranking| r.validate(&labels_for_check),
4847        )
4848        .await;
4849        if self.state.resource_block.is_some() {
4850            self.stop_for_resource()?;
4851            return Ok(false);
4852        }
4853
4854        // Refresh the judgement of every seat that ranked again.
4855        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4856        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4857            let agent_id = seat.agent.clone();
4858            self.state.seats.insert(seat.key.clone(), seat);
4859            let record = &mut self.state.judgements[j];
4860            match res {
4861                Ok((ranking, out)) => {
4862                    record.agent = agent_id;
4863                    record.ranking = ranking.normalized();
4864                    record.reasons = ranking.reasons;
4865                    record.strengths = ranking.strengths;
4866                    record.confidence = ranking.confidence;
4867                    record.failed = None;
4868                    record.duration_ms = out.duration_ms;
4869                    recovered.insert(j);
4870                    self.state.event(
4871                        "recover",
4872                        format!("judge {} ranked again after the limit", j + 1),
4873                    );
4874                }
4875                Err(e) => {
4876                    self.state
4877                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4878                }
4879            }
4880        }
4881
4882        // Re-ask the votes of the seats that recovered a ranking.
4883        let mut vote_jobs = Vec::new();
4884        let mut vote_pos: Vec<usize> = Vec::new();
4885        for &j in &recovered {
4886            let seat_key = format!("judge-{}", j + 1);
4887            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4888            let seat = self.seat(&seat_key, &spec.id);
4889            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4890            let text = if has_context(&spec, &seat, sessions) {
4891                prompt::final_vote(&labels, &language)
4892            } else {
4893                full.clone()
4894            };
4895            vote_jobs.push(SeatJob {
4896                spec,
4897                seat,
4898                prompt: text,
4899                cwd: root.join(seat_key),
4900                timeout,
4901                allow_write: false,
4902                sessions,
4903                artifacts: artifacts.clone(),
4904                stem: format!("vote-judge-{}-recover", j + 1),
4905                handover: Some(full),
4906            });
4907            vote_pos.push(j);
4908        }
4909        let allowed = labels.clone();
4910        let mut vote_losses = Vec::new();
4911        let vote_retries = self.state.config.graph.retries;
4912        let vote_cache = self.state.config.cache_dir();
4913        let ctx = WaveCtx {
4914            carry_seats: false,
4915            pauses: Some((&self.pause, &self.interrupt)),
4916            run: &run_id,
4917            node: "vote",
4918            prompts: &prompts,
4919            cache: vote_cache.as_deref(),
4920            round: None,
4921        };
4922        let votes = ask_json_wave::<FinalVote>(
4923            vote_jobs,
4924            Arc::clone(&self.sem),
4925            vote_retries,
4926            &self.roles.judge_roster,
4927            &ctx,
4928            &mut vote_losses,
4929            &mut self.state,
4930            &move |v: &FinalVote| match v.label() {
4931                Some(c) if allowed.contains(&c) => Ok(()),
4932                other => bail!("vote {other:?} is not one of {allowed:?}"),
4933            },
4934        )
4935        .await;
4936        if self.state.resource_block.is_some() {
4937            self.stop_for_resource()?;
4938            return Ok(false);
4939        }
4940        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4941            let agent_id = seat.agent.clone();
4942            self.state.seats.insert(seat.key.clone(), seat);
4943            match res {
4944                Ok((v, _)) => {
4945                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4946                        rec.vote = v.label();
4947                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4948                    } else {
4949                        self.state.votes.push(VoteRecord {
4950                            judge: j + 1,
4951                            agent: agent_id,
4952                            vote: v.label(),
4953                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4954                            changed: false,
4955                        });
4956                    }
4957                    self.state.event(
4958                        "recover",
4959                        format!("judge {} voted again after the limit", j + 1),
4960                    );
4961                }
4962                Err(e) => {
4963                    self.state
4964                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4965                }
4966            }
4967        }
4968
4969        // A seat that ranked again is present even if its re-vote failed —
4970        // `tally` falls back to the initial ranking's first choice — so clear
4971        // its quota loss. Seats that still fail keep theirs and stay absent.
4972        let recovered_keys: BTreeSet<String> = recovered
4973            .iter()
4974            .map(|&j| format!("judge-{}", j + 1))
4975            .collect();
4976        self.state
4977            .quota
4978            .retain(|q| !recovered_keys.contains(&q.seat));
4979        // A seat that hit the limit again is a fresh loss, not the old one:
4980        // replace the stale entry so the history stays one-per-seat and the
4981        // daemon can tell this attempt's loss from a previous session's.
4982        for loss in judge_losses.into_iter().chain(vote_losses) {
4983            if recovered_keys.contains(&loss.seat) {
4984                continue;
4985            }
4986            self.state.quota.retain(|q| q.seat != loss.seat);
4987            self.state.quota.push(loss);
4988        }
4989
4990        // Recompute the verdict from the refreshed panel.
4991        self.state.tally = None;
4992        self.tally()?;
4993        Ok(self
4994            .state
4995            .tally
4996            .as_ref()
4997            .map(|t| t.met_quorum)
4998            .unwrap_or(false))
4999    }
5000
5001    // ---------------------------------------------------------------- graft
5002
5003    /// Port the strengths the judges attributed to the losing candidates onto
5004    /// the winner, once, between the tally and the review loop.
5005    ///
5006    /// Best-effort by construction: it returns nothing and fails nothing. A
5007    /// skip, a fixer that failed or hit quota, or a round that changed nothing
5008    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
5009    /// the run carries on as if the phase were off. See [`graft`] for the
5010    /// decisions and `AGENTS.md` for the invariants.
5011    ///
5012    /// The one exception is a rollback that could not be confirmed: the
5013    /// rejected commit may still be the winner's tip, so the run is blocked
5014    /// (resumable; the record stays `Running`, which makes a resume retry the
5015    /// restore) instead of reviewing, gating and merging it.
5016    async fn graft(&mut self) -> Result<()> {
5017        let Err(e) = self.graft_inner().await else {
5018            return Ok(());
5019        };
5020        if e.is::<crate::repo_lock::ResourceBlocked>() {
5021            return Err(e);
5022        }
5023        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5024            self.state.status = RunStatus::Blocked;
5025            self.state.event("graft", format!("blocked: {e:#}"));
5026            self.state.save()?;
5027            return Err(e);
5028        }
5029        // An error after the `Running` record was saved may have left the
5030        // fixer's work in the tree: the record must not stay unsettled
5031        // (a later resume would reset past the review fixes), so confirm the
5032        // restore now, or block.
5033        if let Some(rec) = self.state.graft.clone()
5034            && !rec.outcome.settled()
5035        {
5036            if let Some(winner) = self.state.winner().cloned() {
5037                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5038                    if re.is::<crate::repo_lock::ResourceBlocked>() {
5039                        return Err(re);
5040                    }
5041                    self.state.status = RunStatus::Blocked;
5042                    self.state.event("graft", format!("blocked: {re:#}"));
5043                    self.state.save()?;
5044                    return Err(re);
5045                }
5046            }
5047            if let Some(g) = self.state.graft.as_mut() {
5048                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5049            }
5050        }
5051        self.state.event("graft", format!("skipped: {e:#}"));
5052        if let Err(e) = self.state.save() {
5053            tracing::warn!("save after graft: {e:#}");
5054        }
5055        Ok(())
5056    }
5057
5058    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
5059    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
5060    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5061        let _coordination = self.coordinate(wt).await?;
5062        crate::recovery::restore_repair(wt, pre)
5063            .await
5064            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5065    }
5066
5067    /// A skip is a run event, recorded once however often the node is re-entered.
5068    fn graft_skipped(&mut self, why: &str) {
5069        let message = format!("skipped: {why}");
5070        if !self
5071            .state
5072            .events
5073            .iter()
5074            .any(|e| e.node == "graft" && e.message == message)
5075        {
5076            self.state.event("graft", message);
5077        }
5078    }
5079
5080    async fn graft_inner(&mut self) -> Result<()> {
5081        // A round that was started and never finished (crash, kill): the
5082        // fixer may have left anything in the tree, so put the winner back.
5083        if let Some(rec) = self.state.graft.clone()
5084            && !rec.outcome.settled()
5085        {
5086            // Reviews exist only if the graft ended without its record being
5087            // settled (a swallowed error): the tree has moved on since, and
5088            // a reset to `pre` would discard the review fixes.
5089            if self.state.reviews.is_empty()
5090                && let Some(winner) = self.state.winner().cloned()
5091            {
5092                self.graft_restore(&winner.worktree, &rec.pre).await?;
5093            }
5094            if rec.outcome == GraftOutcome::WaitingForRestore {
5095                self.state.graft = None;
5096                self.state.save()?;
5097            } else {
5098                if let Some(g) = self.state.graft.as_mut() {
5099                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5100                }
5101                self.state.event(
5102                    "graft",
5103                    "rolled back: the previous round was interrupted; the winner is untouched",
5104                );
5105                self.state.save()?;
5106                return Ok(());
5107            }
5108        }
5109        let plan = match graft::plan(&self.state) {
5110            Ok(plan) => plan,
5111            Err(skip) => {
5112                if skip != graft::Skip::AlreadyDone {
5113                    self.graft_skipped(skip.reason());
5114                    self.state.save()?;
5115                }
5116                return Ok(());
5117            }
5118        };
5119        let Some(winner) = self.state.winner().cloned() else {
5120            return Ok(());
5121        };
5122        let wt = winner.worktree.clone();
5123        if !git::is_clean(&wt).await? {
5124            self.graft_skipped("the winner's worktree has uncommitted changes");
5125            self.state.save()?;
5126            return Ok(());
5127        }
5128        let pre = git::rev_parse(&wt, "HEAD").await?;
5129        // Saved before the fixer is called: a stop from here on is repaired
5130        // by the branch above on the next entry.
5131        self.state.graft = Some(GraftRecord {
5132            pre: pre.clone(),
5133            post: None,
5134            lines: None,
5135            limit: None,
5136            agent: None,
5137            outcome: GraftOutcome::Running,
5138            requested: graft::requested(&plan),
5139            adoption: None,
5140            adoption_error: None,
5141        });
5142        self.state.event(
5143            "graft",
5144            format!(
5145                "porting strengths of candidate(s) {} onto {}",
5146                plan.strengths
5147                    .iter()
5148                    .map(|(l, _, _)| l.to_string())
5149                    .collect::<Vec<_>>()
5150                    .join(", "),
5151                plan.winner
5152            ),
5153        );
5154        self.state.save()?;
5155
5156        let fix_prompt = prompt::graft(
5157            &self.state.instruction,
5158            plan.winner,
5159            &plan.strengths,
5160            &self.state.config.graph.language,
5161        );
5162        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5163        let sessions = self.state.config.graph.sessions;
5164        let artifacts = agent::artifacts_dir(&self.state.dir());
5165        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5166        let (_, seat, out) = self
5167            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5168                SeatJob {
5169                    prompt: fix_prompt.clone(),
5170                    spec,
5171                    seat,
5172                    cwd: wt.clone(),
5173                    timeout,
5174                    allow_write: true,
5175                    sessions,
5176                    artifacts: artifacts.clone(),
5177                    stem: "graft".to_owned(),
5178                    handover: None,
5179                }
5180            })
5181            .await;
5182        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5183            // Preserve the need to restore earlier fallback edits without
5184            // confusing an unstarted seat with an interrupted invocation.
5185            self.state.graft.as_mut().expect("graft record").outcome =
5186                GraftOutcome::WaitingForRestore;
5187            return self.stop_for_resource();
5188        }
5189        let mut why: Option<String> = None;
5190        match &out {
5191            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5192                why = Some("the fixer produced nothing usable".to_owned());
5193            }
5194            AgentOutcome::Ok(_) => {}
5195            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5196            AgentOutcome::Quota(o) => {
5197                self.state.quota.push(QuotaLoss::from_report(
5198                    seat.key.clone(),
5199                    "graft",
5200                    o.quota.as_ref(),
5201                ));
5202                why = Some("rate limited (quota)".to_owned());
5203            }
5204            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5205                why = Some(format!(
5206                    "the fixer failed: {}",
5207                    e.lines()
5208                        .next()
5209                        .unwrap_or("")
5210                        .chars()
5211                        .take(160)
5212                        .collect::<String>()
5213                ));
5214            }
5215        }
5216        let agent_id = seat.agent.clone();
5217        self.state.seats.insert(seat.key.clone(), seat);
5218
5219        let mut post = None;
5220        let mut lines = None;
5221        let _coordination = self.coordinate(&wt).await?;
5222        if why.is_none() {
5223            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5224            if let Ok(r) = &rescued {
5225                self.state.note_withheld("graft", &r.withheld);
5226            }
5227            let head = git::rev_parse(&wt, "HEAD").await?;
5228            // What is measured must be what gets reviewed and gated: a failed
5229            // rescue, or anything still uncommitted (withheld lockfiles
5230            // aside, which are never part of the tree), rejects the graft.
5231            let dirty = match &rescued {
5232                Ok(r) => git::status_porcelain(&wt)
5233                    .await
5234                    .map(|st| {
5235                        st.lines().any(|l| {
5236                            !r.withheld
5237                                .iter()
5238                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5239                        })
5240                    })
5241                    .unwrap_or(true),
5242                Err(_) => true,
5243            };
5244            if dirty {
5245                why = Some("the fixer left changes that could not be committed".to_owned());
5246            } else if head == pre {
5247                why = Some("the fixer changed nothing".to_owned());
5248            } else {
5249                lines = git::changed_lines(&wt, &pre, &head).await;
5250                post = Some(head);
5251            }
5252        }
5253        let (adoption, adoption_error) = match &out {
5254            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5255            _ => (None, None),
5256        };
5257        let requested = graft::requested(&plan);
5258        let outcome = match why {
5259            None => {
5260                self.state.event(
5261                    "graft",
5262                    format!(
5263                        "applied: {}",
5264                        lines.map_or_else(
5265                            || "changed lines unavailable".to_owned(),
5266                            |n| format!("{n} changed lines")
5267                        )
5268                    ),
5269                );
5270                GraftOutcome::Applied
5271            }
5272            Some(why) => {
5273                // Not settled until the restore is confirmed: the record
5274                // stays `Running`, so a resume retries it.
5275                crate::recovery::restore_repair(&wt, &pre)
5276                    .await
5277                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5278                self.state.event(
5279                    "graft",
5280                    format!("rolled back: {why}; the winner is untouched"),
5281                );
5282                GraftOutcome::RolledBack(why)
5283            }
5284        };
5285        self.state.graft = Some(GraftRecord {
5286            pre,
5287            post,
5288            lines,
5289            limit: None,
5290            agent: Some(agent_id),
5291            outcome,
5292            requested,
5293            adoption,
5294            adoption_error,
5295        });
5296        self.state.save()?;
5297        Ok(())
5298    }
5299
5300    // ----------------------------------------------------------------- fold
5301
5302    async fn fold_losers(&mut self) -> Result<()> {
5303        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5304            return Ok(());
5305        };
5306        if !self.state.candidate_preservation.is_empty() {
5307            bail!("candidate preservation incomplete; refusing cleanup");
5308        }
5309        let repo = self.state.repo.clone();
5310        let _coordination = self.coordinate(&repo).await?;
5311        let mut folded = Vec::new();
5312        for i in 0..self.state.candidates.len() {
5313            let c = &self.state.candidates[i];
5314            if c.label == winner || c.folded {
5315                continue;
5316            }
5317            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5318            remove_preserved_worktree(&repo, &wt).await?;
5319            if git::branch_exists(&repo, &branch).await?
5320                && !git::branch_delete(&repo, &branch).await?
5321            {
5322                bail!("candidate branch removal refused");
5323            }
5324            self.state.candidates[i].folded = true;
5325            self.state.save()?;
5326            folded.push(label.to_string());
5327        }
5328        // The judges are finished; their checkouts are pure cost from here.
5329        let root = self.state.worktree_root();
5330        for j in 1..=self.roles.judges.len() {
5331            let wt = root.join(format!("judge-{j}"));
5332            if wt.exists() {
5333                remove_read_seat_worktree(&mut self.state, &wt).await?;
5334                self.state.save()?;
5335            }
5336        }
5337        // The design-deliberation stage is finished by the time a tally
5338        // exists — same reasoning as the judges above.
5339        if self.state.config.graph.advise {
5340            for k in 1..=self.state.config.graph.advisors {
5341                let wt = root.join(format!("advisor-{k}"));
5342                if wt.exists() {
5343                    remove_read_seat_worktree(&mut self.state, &wt).await?;
5344                    self.state.save()?;
5345                }
5346            }
5347        }
5348        if !folded.is_empty() {
5349            self.state
5350                .event("fold", format!("folded candidates {}", folded.join(", ")));
5351            self.state.save()?;
5352        }
5353        Ok(())
5354    }
5355
5356    // ------------------------------------------------------------ base sync
5357
5358    async fn hold_base_sync(&mut self) -> Result<bool> {
5359        self.hold_base_sync_as(None).await
5360    }
5361
5362    async fn hold_base_sync_kind(
5363        &mut self,
5364        kind: crate::base_sync_recovery::StopKind,
5365    ) -> Result<()> {
5366        self.hold_base_sync_as(Some(kind)).await?;
5367        Ok(())
5368    }
5369
5370    async fn hold_base_sync_as(
5371        &mut self,
5372        kind: Option<crate::base_sync_recovery::StopKind>,
5373    ) -> Result<bool> {
5374        if !self.state.base_sync_blocked() {
5375            self.state.base_sync_recovery.stop = None;
5376            return Ok(false);
5377        }
5378        let head = match self.state.winner() {
5379            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5380                .await
5381                .unwrap_or_default(),
5382            None => String::new(),
5383        };
5384        if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5385            && !authorization.consumed
5386            && authorization.stop.head != head
5387        {
5388            authorization.refuse(format!(
5389                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5390                authorization.stop.head
5391            ));
5392        }
5393        self.state.record_base_sync_stop(&head);
5394        if let Some(kind) = kind {
5395            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5396            if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5397                stop.kind = kind;
5398            }
5399        }
5400        self.state.save()?;
5401        Ok(true)
5402    }
5403
5404    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5405    /// anything verifies it.
5406    ///
5407    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5408    /// read whatever is checked out in the winner's worktree. Left alone that
5409    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5410    /// it when the run *branched* - and a run takes long enough that the base
5411    /// has usually moved by the time it gets here. A gate that ran there
5412    /// answers "green on the commit this run started from", not "green on
5413    /// what is about to land", and the difference showed up three times in
5414    /// one day as a green run whose merge would have reverted a file another
5415    /// pull request had already landed.
5416    ///
5417    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5418    /// `land::Step::Rebase` calls, rather than a second implementation of the
5419    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5420    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5421    /// doc warns about ("two answers to a question nobody notices until a
5422    /// diff is wrong").
5423    ///
5424    /// Foreign-checkout reviews instead merge the base into the owned shadow
5425    /// worktree, preserving the ancestry required for fast-forward delivery.
5426    /// A conflicting merge is aborted and held for operator resolution.
5427    ///
5428    /// A conflict is not the end of the road: the standing rebase is handed
5429    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5430    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5431    /// land). Once it finishes, review and the gate run as usual on the
5432    /// rebased tree, which is where a breakage the new base caused is caught
5433    /// by the ordinary gate-fix round. magi resolves nothing itself.
5434    ///
5435    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5436    /// the current durable recovery interval, bounds how often the base is
5437    /// synchronized during active recovery (lifetime attempts remain evidence).
5438    /// `rebase_fixes` is how many times a *conflict* was given to a fixer. When the fixer cannot finish the
5439    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5440    /// what was tried (rounds spent, paths still conflicted) and the branch
5441    /// and worktree stay exactly as they were - untouched, for a person to
5442    /// look at - which is also what makes re-entering this function
5443    /// afterwards a no-op instead of a second attempt at the same wall. A
5444    /// push failure ends the same way.
5445    async fn sync_to_base(&mut self) -> Result<()> {
5446        if self.state.status == RunStatus::AlreadyInBase {
5447            return Ok(());
5448        }
5449        self.hold_base_sync().await?;
5450        let conflicted = self
5451            .state
5452            .base_sync
5453            .as_ref()
5454            .is_some_and(|s| s.conflict.is_some());
5455        let Some(winner) = self.state.winner().cloned() else {
5456            return Ok(());
5457        };
5458
5459        crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5460
5461        let repo = self.state.repo.clone();
5462        let remote = self.state.config.merge.remote.clone();
5463        let base_branch = self.state.base_branch.clone();
5464        let tracking = format!("{remote}/{base_branch}");
5465
5466        let fetched_base =
5467            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5468        // No network, or the remote never had this branch: the run already
5469        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5470        // otherwise), and one that got this far is not blocked by a fetch
5471        // that fails now. It just has no newer tip to compare against.
5472        let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5473            return Ok(());
5474        };
5475
5476        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5477        self.state.ensure_base_episode();
5478        let mut pending = self
5479            .state
5480            .base_sync_recovery
5481            .operations
5482            .last()
5483            .filter(|o| !o.finished)
5484            .cloned();
5485        let scratch = self.state.dir().join("base-sync");
5486        if git::rebase_in_progress(&scratch).await {
5487            // Older resource stops predate operation reservations. Import the
5488            // standing Git operation, never retarget it to today's main.
5489            if pending.is_none()
5490                && self.state.base_sync_recovery.operations.is_empty()
5491                && !conflicted
5492                && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5493                    .await
5494                    .ok()
5495                    .as_deref()
5496                    == Some(head.as_str())
5497                && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5498                && git::is_clean(&winner.worktree).await?
5499            {
5500                let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5501                if self.state.base_sync.is_none() {
5502                    self.state.base_sync = Some(BaseSync {
5503                        tip: saved_tip.clone(),
5504                        behind: 0,
5505                        attempts: 0,
5506                        conflict: None,
5507                        already_in: None,
5508                    });
5509                }
5510                self.state.reserve_base_operation(&head, &saved_tip);
5511                self.state.event(
5512                    "base-recovery",
5513                    "imported standing legacy rebase as a spent synchronization reservation",
5514                );
5515                self.state.save()?;
5516                pending = self.state.base_sync_recovery.operations.last().cloned();
5517            }
5518            let expected = pending.as_ref().map(|op| op.tip.as_str());
5519            if expected.is_none()
5520                || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5521            {
5522                let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5523                if self.state.base_sync.is_none() {
5524                    self.state.base_sync = Some(BaseSync {
5525                        tip: tip.clone(),
5526                        behind: 0,
5527                        attempts: 0,
5528                        conflict: None,
5529                        already_in: None,
5530                    });
5531                }
5532                self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5533                self.state.event("land", why);
5534                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5535                    .await?;
5536                return Ok(());
5537            }
5538        }
5539        if let Some(op) = &pending {
5540            // A replay may add the pinned base's commits, never foreign work.
5541            let mut replay_proven =
5542                head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5543            if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5544                let wait = repository_wait(&self.state, None);
5545                let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5546                match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5547                    Ok(true) => {
5548                        self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5549                        self.state.save()?;
5550                        git::worktree_remove(&repo, &scratch).await.ok();
5551                        replay_proven = true;
5552                    }
5553                    Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5554                    Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5555                }
5556            }
5557            if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5558                if self.state.foreign_checkout.is_some() {
5559                    replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5560                        .await
5561                        .ok()
5562                        .as_deref()
5563                        == Some(op.head.as_str())
5564                        && git::rev_parse(&repo, &format!("{head}^2"))
5565                            .await
5566                            .ok()
5567                            .as_deref()
5568                            == Some(op.tip.as_str());
5569                } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5570                    .await
5571                    .is_ok_and(|m| m.is_empty())
5572                    && let Ok(added) =
5573                        crate::reconcile::origin_missing(&repo, &op.head, &head).await
5574                {
5575                    replay_proven = true;
5576                    for commit in added {
5577                        if !git::is_ancestor(&repo, &commit, &op.tip).await {
5578                            replay_proven = false;
5579                            break;
5580                        }
5581                    }
5582                    replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5583                        <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5584                }
5585            }
5586            // Never retarget an operation across a crash, even if main moved.
5587            // sync_to_head resets and cleans with -x, so even ignored untracked
5588            // files must stop recovery independently of the old tracked tree.
5589            if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5590                || (head != op.head
5591                    && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5592                        .await?
5593                        .is_empty())
5594                || (!git::is_clean(&winner.worktree).await?
5595                    && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5596                        .await
5597                        .is_ok_and(|o| o.ok()))
5598                || !replay_proven
5599            {
5600                let why = format!(
5601                    "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5602                    op.id
5603                );
5604                let sync = self
5605                    .state
5606                    .base_sync
5607                    .as_mut()
5608                    .context("reserved synchronization has no lifetime accounting")?;
5609                sync.conflict = Some(why.clone());
5610                self.state.event("land", why);
5611                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5612                    .await?;
5613                return Ok(());
5614            }
5615            if head != op.head {
5616                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5617            }
5618            tip = op.tip.clone();
5619        }
5620        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5621        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5622        if pending.is_none()
5623            && !conflicted
5624            && fetched_base
5625            && self
5626                .state
5627                .base_sync_recovery
5628                .episodes
5629                .last()
5630                .is_some_and(|e| e.boundary.is_some())
5631            && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5632            && let Ok(owner) = crate::queue::Queue::open().get(&task)
5633            && owner.execution_runs().last() == Some(&self.state.id)
5634            && owner.status != crate::queue::TaskStatus::Done
5635            && !owner.fresh_start
5636            && owner.review_branch.is_none()
5637            && crate::daemon::compatible_run_instruction(&self.state, &owner)
5638            && let Some(approval) = owner.resumption.as_ref()
5639            && approval.instruction == owner.instruction
5640            && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5641            && git::is_clean(&winner.worktree).await?
5642            && self.state.renew_base_episode(&task, approval, &head, &tip)
5643        {
5644            self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5645            self.state.save()?;
5646        }
5647
5648        // Before any rebase, and before a recorded conflict is honoured: a
5649        // branch whose change reached the base under other commit ids has
5650        // nothing to rebase and nothing to conflict with, and a run that
5651        // already stopped on that phantom conflict recovers here on resume.
5652        // `behind == 0` with head == tip is a branch the base has since taken
5653        // in whole, whether or not a conflict was ever recorded: the ancestry
5654        // proof must still run (`classify` ignores a head still on the start
5655        // commit).
5656        if (behind > 0 || conflicted || head == tip)
5657            && self
5658                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5659                .await?
5660        {
5661            return Ok(());
5662        }
5663        if conflicted && self.state.base_recovery_pending() {
5664            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5665            let owner =
5666                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5667            let owner_valid = owner.as_ref().is_ok_and(|t| {
5668                t.execution_runs().last() == Some(&self.state.id)
5669                    && t.status != crate::queue::TaskStatus::Done
5670                    && !t.fresh_start
5671                    && t.review_branch.is_none()
5672            });
5673            let invalidation = if !owner_valid {
5674                Some("latest task ownership changed")
5675            } else if !owner.as_ref().is_ok_and(|t| {
5676                authorization
5677                    .generation
5678                    .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5679            }) {
5680                Some("owner approval generation changed")
5681            } else if authorization.question.as_ref().is_some_and(|question| {
5682                !crate::base_recovery_control::decision_current(
5683                    &crate::queue::Queue::at(crate::run::home().join("queue")),
5684                    &self.state,
5685                    &authorization.task,
5686                    question,
5687                )
5688            }) {
5689                Some("recovery decision ownership, routing or stop evidence changed")
5690            } else if authorization.request.is_some()
5691                && !owner
5692                    .as_ref()
5693                    .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5694            {
5695                Some("owner decision changed")
5696            } else if let Some(scope) = &authorization.scope {
5697                if scope.tip != tip {
5698                    Some("fetched base changed")
5699                } else if !self
5700                    .state
5701                    .base_sync_recovery
5702                    .episodes
5703                    .last()
5704                    .is_some_and(|e| e.id == scope.episode)
5705                {
5706                    Some("active synchronization interval changed")
5707                } else if !owner
5708                    .as_ref()
5709                    .is_ok_and(|t| t.instruction == scope.instruction)
5710                {
5711                    Some("task instruction changed")
5712                } else {
5713                    None
5714                }
5715            } else {
5716                None
5717            };
5718            if let Some(reason) = invalidation {
5719                let authorization = self
5720                    .state
5721                    .base_sync_recovery
5722                    .authorizations
5723                    .last_mut()
5724                    .unwrap();
5725                authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5726                self.state.save()?;
5727                return Ok(());
5728            }
5729            if !fetched_base
5730                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5731                || !git::is_clean(&winner.worktree).await?
5732                || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5733            {
5734                return Ok(());
5735            }
5736        }
5737        let recovery = conflicted && self.state.consume_base_recovery(&head);
5738        if conflicted {
5739            // Consumption (including a refused changed HEAD) is durable before Git.
5740            self.state.save()?;
5741            if !recovery {
5742                return Ok(());
5743            }
5744        }
5745
5746        if behind == 0 {
5747            // A fixer-finished rebase moves the branch ref before the
5748            // winner's worktree is told (`sync_to_head` below). A run that
5749            // died in between resumes here with `behind == 0` and a tree still
5750            // holding the pre-rebase files, which review and the gate would
5751            // then read. That state is exactly: HEAD moved off the tip the
5752            // rebase started from, yet the tree is still identical to that
5753            // tip. A tree with edits of its own differs from it, so nothing
5754            // is thrown away.
5755            if let Some(from) = self
5756                .state
5757                .rebase_fixes
5758                .iter()
5759                .rev()
5760                .find_map(|r| r.from.clone())
5761                && from != head
5762                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5763                    .await
5764                    .is_ok_and(|o| o.ok())
5765            {
5766                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5767            }
5768            // An earlier attempt may have rebased the branch locally and died
5769            // before pushing it (only the fresh-rebase arm below pushes).
5770            // Publish it now, so the plain push at PR time is not refused as
5771            // a non-fast-forward. A run already holding a recorded conflict
5772            // never reaches here; that case is out of scope.
5773            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5774            if let Some(why) = &conflict {
5775                self.state.status = RunStatus::Blocked;
5776                self.state.event("land", why.clone());
5777            }
5778            self.state.base_sync = Some(BaseSync {
5779                tip,
5780                behind: 0,
5781                attempts,
5782                conflict,
5783                already_in: None,
5784            });
5785            if pending.is_some() {
5786                self.state.finish_base_operation(Some(&head));
5787            }
5788            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5789                .await?;
5790            self.state.save()?;
5791            return Ok(());
5792        }
5793
5794        if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5795            let why = format!(
5796                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5797                 synchronization(s); synchronizing again would only race it",
5798                winner.branch
5799            );
5800            self.state.status = RunStatus::Blocked;
5801            self.state.base_sync = Some(BaseSync {
5802                tip,
5803                behind,
5804                attempts,
5805                conflict: Some(why.clone()),
5806                already_in: None,
5807            });
5808            self.state.event("land", why);
5809            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5810                .await?;
5811            self.state.save()?;
5812            return Ok(());
5813        }
5814
5815        if self.state.foreign_checkout.is_some() {
5816            // Delivery to the operator's checkout is fast-forward only. A
5817            // rebase here would discard its saved head from our ancestry.
5818            let wait = repository_wait(&self.state, None);
5819            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5820            if self.state.base_sync.is_none() {
5821                self.state.base_sync = Some(BaseSync {
5822                    tip: tip.clone(),
5823                    behind,
5824                    attempts: 0,
5825                    conflict: None,
5826                    already_in: None,
5827                });
5828            }
5829            self.state.reserve_base_operation(&head, &tip);
5830            self.state.save()?;
5831            let merged = git::git_raw(
5832                &winner.worktree,
5833                &[
5834                    "-c",
5835                    "core.hooksPath=/dev/null",
5836                    "merge",
5837                    "--no-ff",
5838                    "--no-autostash",
5839                    "--no-edit",
5840                    "-m",
5841                    &format!("Merge {tracking} before verifying"),
5842                    &tip,
5843                ],
5844            )
5845            .await?;
5846            let conflict = if merged.ok() {
5847                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5848                self.state.event(
5849                    "land",
5850                    format!("merged {tracking} into {} before verifying", winner.branch),
5851                );
5852                None
5853            } else {
5854                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5855                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5856                }
5857                let why = format!(
5858                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5859                    winner.branch,
5860                    merged.stderr.chars().take(600).collect::<String>()
5861                );
5862                self.state.status = RunStatus::Blocked;
5863                self.state.event("land", &why);
5864                Some(why)
5865            };
5866            let synced = conflict.is_none();
5867            let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5868            self.state
5869                .finish_base_operation(synced.then_some(fresh.as_str()));
5870            let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5871            self.state.base_sync = Some(BaseSync {
5872                tip,
5873                behind: if synced { 0 } else { behind },
5874                attempts,
5875                conflict,
5876                already_in: None,
5877            });
5878            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5879                .await?;
5880            self.state.save()?;
5881            return Ok(());
5882        }
5883
5884        self.state.event(
5885            "land",
5886            format!(
5887                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5888                winner.branch
5889            ),
5890        );
5891        self.state.save()?;
5892
5893        // The remote's copy of the branch, read now and only if the fetch
5894        // really succeeded (a stale tracking ref must never pin a lease). It is
5895        // pushed over after a rebase only when it is a commit this branch
5896        // already contains, by ancestry or by patch (an earlier rebase of ours
5897        // that never reached the remote): anything else is somebody else's work.
5898        let branch_tracking = format!("{remote}/{}", winner.branch);
5899        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5900        let remote_tip = if self.state.task_pr_branch().is_none()
5901            && matches!(&fetched_branch, Ok(o) if o.ok())
5902        {
5903            git::rev_parse(&repo, &branch_tracking).await.ok()
5904        } else {
5905            None
5906        };
5907        // A remote tip this branch does not contain is somebody else's work:
5908        // rebasing would leave a local tip that can never be pushed. Stop
5909        // before touching anything and say so.
5910        if let Some(theirs) = &remote_tip
5911            && !git::is_ancestor(&repo, theirs, &head).await
5912            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5913                .await
5914                .is_ok_and(|missing| missing.is_empty())
5915        {
5916            let why = format!(
5917                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5918                 them",
5919                short(theirs),
5920                winner.branch
5921            );
5922            self.state.status = RunStatus::Blocked;
5923            self.state.base_sync = Some(BaseSync {
5924                tip,
5925                behind,
5926                attempts,
5927                conflict: Some(why.clone()),
5928                already_in: None,
5929            });
5930            self.state.event("land", why);
5931            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5932                .await?;
5933            self.state.save()?;
5934            return Ok(());
5935        }
5936
5937        if self.state.base_sync.is_none() {
5938            self.state.base_sync = Some(BaseSync {
5939                tip: tip.clone(),
5940                behind,
5941                attempts: 0,
5942                conflict: None,
5943                already_in: None,
5944            });
5945        }
5946        let scratch = self.state.dir().join("base-sync");
5947        if let Err(error) = crate::rebase::prepare(&mut self.state, &scratch, &winner.branch).await
5948        {
5949            if error.is::<crate::repo_lock::ResourceBlocked>() {
5950                return Ok(());
5951            }
5952            return Err(error);
5953        }
5954        self.state.reserve_base_operation(&head, &tip);
5955        self.state.save()?;
5956        let rebased =
5957            match crate::rebase::rebase_prepared(&mut self.state, &scratch, &winner.branch, &tip)
5958                .await
5959            {
5960                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5961                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5962                Err(e) => Err(e),
5963            };
5964        let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5965        let stop_kind = match &rebased {
5966            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5967            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5968            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5969        };
5970        match rebased {
5971            Ok(None) => {
5972                // The branch ref moved, but a worktree that already had it
5973                // checked out (the winner's) was not told; sync its index and
5974                // files before anything reads them.
5975                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5976                let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5977                self.state.finish_base_operation(Some(&fresh));
5978                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5979                let mut conflict = None;
5980                if let Some(pinned) = &remote_tip {
5981                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5982                    match pushed {
5983                        Ok(o) if o.ok() => self.state.event(
5984                            "land",
5985                            format!("pushed rebased {} to {remote}", winner.branch),
5986                        ),
5987                        Ok(o) => {
5988                            conflict = Some(format!(
5989                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5990                                winner.branch,
5991                                short(pinned),
5992                                o.stderr.chars().take(600).collect::<String>()
5993                            ));
5994                        }
5995                        Err(e) => {
5996                            conflict = Some(format!(
5997                                "rebased {} locally but could not push it: {e:#}",
5998                                winner.branch
5999                            ));
6000                        }
6001                    }
6002                }
6003                if let Some(why) = &conflict {
6004                    self.state.status = RunStatus::Blocked;
6005                    self.state.event("land", why.clone());
6006                }
6007                self.state.base_sync = Some(BaseSync {
6008                    tip: tip.clone(),
6009                    behind: 0,
6010                    attempts,
6011                    conflict,
6012                    already_in: None,
6013                });
6014                self.state
6015                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
6016            }
6017            Ok(Some(conflict)) => {
6018                self.state.finish_base_operation(None);
6019                let why = format!(
6020                    "{} conflicts with {tracking} and did not rebase: {}",
6021                    winner.branch,
6022                    conflict.chars().take(600).collect::<String>()
6023                );
6024                self.state.status = RunStatus::Blocked;
6025                self.state.base_sync = Some(BaseSync {
6026                    tip,
6027                    behind,
6028                    attempts,
6029                    conflict: Some(why.clone()),
6030                    already_in: None,
6031                });
6032                self.state.event("land", why);
6033            }
6034            // A refused Git write or a busy repository: nothing is wrong with
6035            // the branch. The scratch rebase stays standing and the run
6036            // resumes into it (`base_sync` must not claim a conflict).
6037            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6038                self.state.save()?;
6039                return Ok(());
6040            }
6041            Err(e) => {
6042                self.state.finish_base_operation(None);
6043                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6044                self.state.status = RunStatus::Blocked;
6045                self.state.base_sync = Some(BaseSync {
6046                    tip,
6047                    behind,
6048                    attempts,
6049                    conflict: Some(why.clone()),
6050                    already_in: None,
6051                });
6052                self.state.event("land", why);
6053            }
6054        }
6055        if !self.state.base_sync_blocked() {
6056            self.state.base_sync_recovery.stop = None;
6057        }
6058        self.hold_base_sync_kind(stop_kind).await?;
6059        self.state.save()?;
6060        Ok(())
6061    }
6062
6063    /// Push a branch an earlier attempt rebased locally but never published,
6064    /// pinned to the remote tip read right after a successful fetch. Returns
6065    /// the reason when the run must stop; `None` when there was nothing to do
6066    /// (no remote copy, the same tip, or a remote copy this branch already
6067    /// contains, which the PR-time push fast-forwards) or the push succeeded.
6068    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6069        // Task competitions publish only after the PR text gate, to their
6070        // public head; never resurrect a candidate branch on the remote.
6071        if self.state.task_pr_branch().is_some() {
6072            return None;
6073        }
6074        let repo = self.state.repo.clone();
6075        let remote = self.state.config.merge.remote.clone();
6076        let fetched = git::fetch(&repo, &remote, branch).await;
6077        if !matches!(&fetched, Ok(o) if o.ok()) {
6078            return None;
6079        }
6080        let branch_tracking = format!("{remote}/{branch}");
6081        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6082        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6083            return None;
6084        }
6085        if !crate::reconcile::origin_missing(&repo, head, &theirs)
6086            .await
6087            .is_ok_and(|missing| missing.is_empty())
6088        {
6089            return Some(format!(
6090                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6091                 them",
6092                short(&theirs)
6093            ));
6094        }
6095        match git::push_pinned(&repo, &remote, branch, &theirs).await {
6096            Ok(o) if o.ok() => {
6097                self.state
6098                    .event("land", format!("pushed rebased {branch} to {remote}"));
6099                None
6100            }
6101            Ok(o) => Some(format!(
6102                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6103                 someone may have pushed): {}",
6104                short(&theirs),
6105                o.stderr.chars().take(600).collect::<String>()
6106            )),
6107            Err(e) => Some(format!(
6108                "{branch} is rebased locally but could not be pushed: {e:#}"
6109            )),
6110        }
6111    }
6112
6113    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
6114    /// is already on `tip` under other commit ids ([`crate::already`]); returns
6115    /// whether it did.
6116    ///
6117    /// Checked only when the base is ahead of the branch. A failing check is
6118    /// "not proven" - the ordinary rebase path then decides - never a reason to
6119    /// stop the run.
6120    ///
6121    /// The remote copy of the branch is held to the same standard as the local
6122    /// one: if it carries a tip this worktree does not, that tip must itself be
6123    /// proven in the base, or nothing is settled (a pull request would
6124    /// otherwise be closed over commits nobody checked). The pull request is
6125    /// closed *before* the terminal status is saved; if that fails for a
6126    /// reason other than a refusal (no network, a `gh` error) the run is left
6127    /// `Blocked` with the reason as its conflict, which a resume retries -
6128    /// the same recovery a phantom conflict gets.
6129    async fn settle_already_in(
6130        &mut self,
6131        branch: &str,
6132        tip: &str,
6133        head: &str,
6134        attempts: usize,
6135        behind: usize,
6136    ) -> Result<bool> {
6137        let repo = self.state.repo.clone();
6138        let remote = self.state.config.merge.remote.clone();
6139        let start = self.state.base_commit.clone();
6140        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6141            Ok(Some(e)) => e,
6142            Ok(None) => return Ok(false),
6143            Err(e) => {
6144                tracing::warn!("already-in-base check for {branch}: {e:#}");
6145                return Ok(false);
6146            }
6147        };
6148        let mut verified = vec![head.to_owned()];
6149        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6150        let fetched = git::fetch(&repo, &remote, &public_branch).await;
6151        if matches!(&fetched, Ok(o) if o.ok())
6152            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6153            && theirs != head
6154        {
6155            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6156                Ok(Some(_)) => verified.push(theirs),
6157                _ => return Ok(false),
6158            }
6159        }
6160        let base_branch = self.state.base_branch.clone();
6161        let message = format!(
6162            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6163             land",
6164            evidence.names(),
6165            evidence.proof.as_str()
6166        );
6167        let closed =
6168            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6169                .await;
6170        match closed {
6171            Ok(Ok(url)) => self
6172                .state
6173                .event("land", format!("closed {url}: superseded on {base_branch}")),
6174            Ok(Err(why)) => self
6175                .state
6176                .event("land", format!("did not close a pull request: {why}")),
6177            Err(e) => {
6178                let why = format!(
6179                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
6180                     not be closed ({e:#}); resume to retry"
6181                );
6182                self.state.status = RunStatus::Blocked;
6183                self.state.base_sync = Some(BaseSync {
6184                    tip: tip.to_owned(),
6185                    behind,
6186                    attempts,
6187                    conflict: Some(why.clone()),
6188                    already_in: None,
6189                });
6190                self.state.event("land", why);
6191                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6192                    .await?;
6193                self.state.save()?;
6194                return Ok(true);
6195            }
6196        }
6197        self.state.status = RunStatus::AlreadyInBase;
6198        self.state.base_sync_recovery.stop = None;
6199        self.state.base_sync = Some(BaseSync {
6200            tip: tip.to_owned(),
6201            behind,
6202            attempts,
6203            conflict: None,
6204            already_in: Some(evidence),
6205        });
6206        self.state.event("land", message);
6207        self.state.save()?;
6208        self.settle_questions();
6209        Ok(true)
6210    }
6211
6212    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
6213    /// last landed the winner on, once it has run, else the commit the run
6214    /// branched from.
6215    ///
6216    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
6217    /// and `vote` all happen before there is a winner to rebase, so they
6218    /// compare every candidate against the branch point on purpose, and a
6219    /// base that moves after they are already done cannot change an answer
6220    /// they already gave.
6221    fn landing_base(&self) -> String {
6222        self.state
6223            .base_sync
6224            .as_ref()
6225            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6226    }
6227
6228    // ------------------------------------------------------- operator fix
6229
6230    /// Route specific, already-recorded review findings to a fixer for a
6231    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
6232    /// entry point.
6233    ///
6234    /// Distinct from `review_loop`'s own fix step in three ways: it never
6235    /// runs a reviewer wave, it never spends review-round budget, and what
6236    /// happened is recorded as an [`OperatorFixRequest`] appended to
6237    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
6238    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
6239    /// and vote must never be rewritten to look like a manufactured blocking
6240    /// verdict.
6241    ///
6242    /// Only meaningful once review has actually concluded: `Ready` (handed
6243    /// off with findings still open, or simply concluded clean while minor
6244    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
6245    /// gate failed). Everything else is refused: a run still in progress
6246    /// should simply be resumed, and a `Merged` run's branch has already
6247    /// landed — reopening *this* run's own record cannot change that, so the
6248    /// answer there is a fresh `magi review <branch>`.
6249    ///
6250    /// A real commit here re-verifies through a fresh, ordinary review-only
6251    /// run on the same branch ([`Self::review`]) rather than reopening this
6252    /// run's own `review_loop`: once any round in this run's history went
6253    /// clean, `review_conclusion` treats that as permanent by design (the
6254    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
6255    /// way to force one more genuine reviewer wave out of *this* run without
6256    /// either rewriting history or weakening that guarantee for every other
6257    /// caller. A review-only run costs nothing extra — no implementation, no
6258    /// judging, no vote — and exercises the exact same review → verify →
6259    /// gate → (human) merge path, unmodified.
6260    pub async fn fix_selected(
6261        &mut self,
6262        ids: &[String],
6263        reason: &str,
6264        allow_stale: bool,
6265    ) -> Result<()> {
6266        let reason = reason.trim();
6267        if reason.is_empty() {
6268            bail!("a fix request needs a reason — that is the operator's own record of why");
6269        }
6270        if ids.is_empty() {
6271            bail!("no finding id given");
6272        }
6273        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6274            bail!(
6275                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6276                 has already concluded — can be given a targeted fix. A run still \
6277                 in progress should simply be resumed; a `merged` run's branch has \
6278                 already landed, so its answer is a fresh `magi review <branch>`, \
6279                 not reopening this run's own record",
6280                self.state.id,
6281                self.state.status.as_str()
6282            );
6283        }
6284        // Reassignment shares this claim through its atomic task save. Check
6285        // the canonical run identity inside it, before preflight or any await.
6286        let _claim = FixClaim::acquire(&self.state.dir())?;
6287        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6288        let Some(winner) = self.state.winner().cloned() else {
6289            bail!("run {} has no winning candidate to fix", self.state.id);
6290        };
6291        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6292            bail!(
6293                "branch `{}` no longer exists; this run cannot be extended",
6294                winner.branch
6295            );
6296        }
6297        let home = crate::run::home();
6298        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6299            bail!(
6300                "run {} is currently being worked on by another magi process",
6301                self.state.id
6302            );
6303        }
6304        // Resolve every id before spending anything — an unknown id refuses
6305        // the whole request rather than silently dropping it — and dedup
6306        // while keeping the operator's own order.
6307        let mut seen = BTreeSet::new();
6308        let mut findings = Vec::new();
6309        let mut missing = Vec::new();
6310        for id in ids {
6311            if !seen.insert(id.clone()) {
6312                continue;
6313            }
6314            match self.state.finding(id) {
6315                Some((round, rec, f)) => findings.push(OperatorFixFinding {
6316                    id: f.id.clone(),
6317                    severity: f.severity,
6318                    reviewer_vote: rec.vote,
6319                    round: round.round,
6320                    round_head: round.head.clone(),
6321                    reviewer: rec.reviewer,
6322                    agent: rec.agent.clone(),
6323                    file: f.file.clone(),
6324                    line: f.line,
6325                    title: f.title.clone(),
6326                    detail: f.detail.clone(),
6327                    outcome: OperatorFixOutcome::Pending,
6328                }),
6329                None => missing.push(id.clone()),
6330            }
6331        }
6332        if !missing.is_empty() {
6333            bail!(
6334                "unknown finding id(s): {}; nothing was changed",
6335                missing.join(", ")
6336            );
6337        }
6338
6339        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6340        let stale_details: Vec<(String, String)> = findings
6341            .iter()
6342            .filter(|f| f.round_head != head_at_request)
6343            .map(|f| (f.id.clone(), f.round_head.clone()))
6344            .collect();
6345        let stale = !stale_details.is_empty();
6346        if stale && !allow_stale {
6347            bail!(
6348                "the branch has moved since some finding(s) were raised — {} — now \
6349                 at {}; pass --allow-stale to fix anyway, or re-run review first",
6350                stale_details
6351                    .iter()
6352                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6353                    .collect::<Vec<_>>()
6354                    .join(", "),
6355                short(&head_at_request)
6356            );
6357        }
6358
6359        let request = OperatorFixRequest {
6360            requested_at: Timestamp::now(),
6361            reason: reason.to_owned(),
6362            findings,
6363            head_at_request: head_at_request.clone(),
6364            allow_stale,
6365            stale,
6366            fix: None,
6367            result_head: None,
6368            follow_up_review_run: None,
6369        };
6370        self.state.event(
6371            "fix",
6372            format!(
6373                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6374                request.findings.len(),
6375                request
6376                    .findings
6377                    .iter()
6378                    .map(|f| f.id.as_str())
6379                    .collect::<Vec<_>>()
6380                    .join(", "),
6381            ),
6382        );
6383        // Recorded now, before any worktree work or the fixer call itself —
6384        // and re-saved at each checkpoint below: a crash at any point after
6385        // this (mid fixer call, mid follow-up review) must not lose the fact
6386        // that this was requested, for which findings, and why. Everything
6387        // past this point reads and writes through `request_index` rather
6388        // than a local variable, since `request` itself is moved here.
6389        self.state.operator_fixes.push(request);
6390        self.state.save()?;
6391        let request_index = self.state.operator_fixes.len() - 1;
6392
6393        // A fresh, dedicated worktree for this one call, never the winner's
6394        // own worktree in place: that one may already be gone (folded away),
6395        // and reusing it in place would leave the branch checked out there
6396        // when the follow-up review below tries to check it out again. Freed
6397        // immediately after, either way — but only once confirmed clean:
6398        // `worktree_remove` is a `git worktree remove --force`, which would
6399        // otherwise discard uncommitted work left there by the operator or
6400        // another process before this had a chance to even look at it.
6401        let repo = self.state.repo.clone();
6402        let coordination = self.coordinate(&repo).await?;
6403        if winner.worktree.exists() {
6404            // Lockfiles a rescue commit withheld stay untracked on purpose and
6405            // are already recorded; they are not the operator's work to protect.
6406            let dirty = git::git(
6407                &winner.worktree,
6408                &["status", "--porcelain", "--untracked-files=all"],
6409            )
6410            .await?;
6411            let only_withheld = dirty.lines().all(|l| {
6412                l.strip_prefix("?? ")
6413                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6414            });
6415            if !only_withheld {
6416                bail!(
6417                    "`{}` has uncommitted changes; refusing to touch it — commit or \
6418                     discard them first",
6419                    winner.worktree.display()
6420                );
6421            }
6422            git::worktree_remove(&self.state.repo, &winner.worktree)
6423                .await
6424                .ok();
6425        }
6426        let fix_worktree = self.state.worktree_root().join("operator-fix");
6427        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6428        git::git(
6429            &self.state.repo,
6430            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6431        )
6432        .await
6433        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6434        if !git::is_clean(&fix_worktree).await? {
6435            git::worktree_remove(&self.state.repo, &fix_worktree)
6436                .await
6437                .ok();
6438            bail!(
6439                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6440                winner.branch
6441            );
6442        }
6443        let repo_for_setup = self.state.repo.clone();
6444        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6445            Ok(()) => {
6446                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6447            }
6448            Err(e) => Err(e),
6449        };
6450        if let Err(e) = setup {
6451            git::worktree_remove(&self.state.repo, &fix_worktree)
6452                .await
6453                .ok();
6454            return Err(e);
6455        }
6456
6457        // This explicit operator request retries availability independently
6458        // of execute(); the previous call has already saved and cleaned up.
6459        self.state.resource_block = None;
6460
6461        let run_id = self.state.id.clone();
6462        let prompts = self.state.config.prompts.clone();
6463        let language = self.state.config.graph.language.clone();
6464        let sessions = self.state.config.graph.sessions;
6465        let artifacts = agent::artifacts_dir(&self.state.dir());
6466        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6467            .findings
6468            .iter()
6469            .map(|f| Finding {
6470                id: f.id.clone(),
6471                severity: f.severity,
6472                file: f.file.clone(),
6473                line: f.line,
6474                title: f.title.clone(),
6475                detail: f.detail.clone(),
6476            })
6477            .collect();
6478        let fix_prompt = prompt::operator_fix_in(
6479            &self.state.instruction,
6480            &finding_list,
6481            reason,
6482            &stale_details,
6483            &head_at_request,
6484            &language,
6485            &self.state.config.graph.github_language,
6486        );
6487        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6488        drop(coordination);
6489        let (job, seat, out) = self
6490            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6491                prompt: fix_prompt.clone(),
6492                spec,
6493                seat,
6494                cwd: fix_worktree.clone(),
6495                timeout,
6496                allow_write: true,
6497                sessions,
6498                artifacts: artifacts.clone(),
6499                stem: "operator-fix".to_owned(),
6500                handover: None,
6501            })
6502            .await;
6503        let agent_id = seat.agent.clone();
6504
6505        let mut fix = FixRecord {
6506            agent: agent_id,
6507            addressed: Vec::new(),
6508            rejected: Vec::new(),
6509            notes: String::new(),
6510            committed: false,
6511            failed: None,
6512            duration_ms: 0,
6513            continuation: None,
6514        };
6515        let mut final_seat = seat.clone();
6516        match out {
6517            AgentOutcome::Ok(o) => {
6518                fix.duration_ms = o.duration_ms;
6519                let parsed = verdict::extract_json::<FixReport>(&o.text);
6520                let incomplete_reason = match &parsed {
6521                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6522                        "the reply parsed, but it reported a command whose own CLI \
6523                         never confirmed an exit status"
6524                            .to_owned(),
6525                    ),
6526                    Ok(_) => None,
6527                    Err(e) => Some(e.to_string()),
6528                };
6529                match incomplete_reason {
6530                    None => {
6531                        let report = parsed.expect("checked Ok above");
6532                        fix.addressed = report.addressed;
6533                        fix.rejected = report.rejected;
6534                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6535                    }
6536                    Some(reason) => {
6537                        let (resumed_seat, resolved, failure, cont) = self
6538                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6539                            .await;
6540                        fix.duration_ms += cont.cumulative_wait_ms;
6541                        fix.continuation = Some(cont);
6542                        final_seat = resumed_seat;
6543                        match resolved {
6544                            Some(report) => {
6545                                fix.addressed = report.addressed;
6546                                fix.rejected = report.rejected;
6547                                fix.notes =
6548                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6549                            }
6550                            None => fix.failed = failure,
6551                        }
6552                    }
6553                }
6554            }
6555            AgentOutcome::Dropped(o) => {
6556                fix.duration_ms = o.duration_ms;
6557                let why = o
6558                    .dropped
6559                    .as_ref()
6560                    .map(|d| d.why.as_str())
6561                    .unwrap_or("the CLI ended the stream without delivering its answer");
6562                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6563            }
6564            AgentOutcome::Quota(o) => {
6565                self.state.quota.push(QuotaLoss::from_report(
6566                    final_seat.key.clone(),
6567                    "fix",
6568                    o.quota.as_ref(),
6569                ));
6570                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6571            }
6572            AgentOutcome::Failed(e) => fix.failed = Some(e),
6573            AgentOutcome::ResourceBlocked(e) => {
6574                fix.failed = Some(format!("the fixer was not started: {e}"));
6575            }
6576        }
6577        // Even a deferred call must rescue edits, release the worktree and
6578        // finalize its record below so an explicit `magi fix` can retry.
6579        if fix.continuation.is_none() {
6580            fix.continuation = Some(ContinuationRecord::not_needed());
6581        }
6582        self.state.seats.insert(final_seat.key.clone(), final_seat);
6583
6584        let rescue_message = format!(
6585            "magi: operator-selected fix ({}) (uncommitted work)",
6586            self.state.operator_fixes[request_index]
6587                .findings
6588                .iter()
6589                .map(|f| f.id.as_str())
6590                .collect::<Vec<_>>()
6591                .join(", ")
6592        );
6593        let repo = self.state.repo.clone();
6594        let coordination = self.coordinate(&repo).await?;
6595        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6596            self.state.note_withheld("fix", &r.withheld);
6597        }
6598        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6599        fix.committed = after != head_at_request;
6600        git::worktree_remove(&self.state.repo, &fix_worktree)
6601            .await
6602            .ok();
6603        drop(coordination);
6604
6605        self.state.event(
6606            "fix",
6607            match &fix.failed {
6608                Some(reason) => format!(
6609                    "operator fix: adoption report was lost ({reason}); {}",
6610                    if fix.committed {
6611                        "committed"
6612                    } else {
6613                        "NO new commit"
6614                    }
6615                ),
6616                None => format!(
6617                    "operator fix: {} addressed, {} rejected, {}",
6618                    fix.addressed.len(),
6619                    fix.rejected.len(),
6620                    if fix.committed {
6621                        "committed"
6622                    } else {
6623                        "NO new commit"
6624                    }
6625                ),
6626            },
6627        );
6628
6629        // Every selected finding gets an outcome — never left `Pending` once
6630        // the fixer's own turn is over. A report that never came back at all
6631        // marks every one of them `Unreported`, not silently "not addressed":
6632        // quota, a dropped stream, or an exhausted continuation are gaps in
6633        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6634        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6635        for f in &mut self.state.operator_fixes[request_index].findings {
6636            f.outcome = if fix.failed.is_some() {
6637                OperatorFixOutcome::Unreported
6638            } else if fix.addressed.contains(&f.id) {
6639                OperatorFixOutcome::Addressed
6640            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6641                OperatorFixOutcome::Rejected { why: r.why.clone() }
6642            } else {
6643                OperatorFixOutcome::Unreported
6644            };
6645        }
6646
6647        let committed = fix.committed;
6648        if committed {
6649            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6650        }
6651        self.state.operator_fixes[request_index].fix = Some(fix);
6652        // Saved again now that the fixer's own outcome is final, on top of
6653        // the save right after the request was first pushed above.
6654        self.state.save()?;
6655        if self.state.resource_block.is_some() {
6656            return self.stop_for_resource();
6657        }
6658
6659        if committed {
6660            self.state.event(
6661                "fix",
6662                format!(
6663                    "operator fix committed {}; opening a follow-up review-only run",
6664                    short(&after)
6665                ),
6666            );
6667            // The operator asked for the fix, and the follow-up serves whatever
6668            // task the run it follows served.
6669            let origin =
6670                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6671            match Self::review(
6672                &self.state.repo,
6673                &winner.branch,
6674                self.state.config.clone(),
6675                origin,
6676            )
6677            .await
6678            {
6679                Ok(mut follow_up) => {
6680                    follow_up.state.event(
6681                        "start",
6682                        format!(
6683                            "requested by an operator fix on run {} for finding(s) {}",
6684                            self.state.id,
6685                            self.state.operator_fixes[request_index]
6686                                .findings
6687                                .iter()
6688                                .map(|f| f.id.as_str())
6689                                .collect::<Vec<_>>()
6690                                .join(", "),
6691                        ),
6692                    );
6693                    follow_up.state.save()?;
6694                    let follow_up_id = follow_up.state.id.clone();
6695                    // The follow-up is a run like any other: it belongs to the
6696                    // task of the run it follows, or to one filed for it.
6697                    let adopted = match crate::direct::adopt(
6698                        &follow_up.state,
6699                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6700                    ) {
6701                        Ok(a) => a,
6702                        Err(e) => {
6703                            // An ownerless run must not spend agent calls; it
6704                            // stays saved, and `magi run --resume` adopts it.
6705                            self.state.event(
6706                                "fix",
6707                                format!(
6708                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6709                                ),
6710                            );
6711                            self.state.save()?;
6712                            return Ok(());
6713                        }
6714                    };
6715                    let executed = follow_up.execute().await;
6716                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6717                    if let Some(a) = adopted {
6718                        a.finish(&follow_up.state, executed);
6719                    }
6720                    if let Some(e) = failure {
6721                        self.state.event(
6722                            "fix",
6723                            format!(
6724                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6725                            ),
6726                        );
6727                    }
6728                    self.state.operator_fixes[request_index].follow_up_review_run =
6729                        Some(follow_up_id);
6730                }
6731                Err(e) => {
6732                    self.state.event(
6733                        "fix",
6734                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6735                    );
6736                }
6737            }
6738            self.state.save()?;
6739        }
6740
6741        Ok(())
6742    }
6743
6744    // --------------------------------------------------------------- review
6745
6746    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6747    /// names, in order, each at most once, moving on only when the call
6748    /// advances (an error, a quota hit or nothing usable - the same decision
6749    /// point `agent::chain_advances` is for the other chained roles).
6750    ///
6751    /// Each agent gets its own seat from `Runner::seat`: the same agent
6752    /// continues its conversation (the winner's own implementer seat when it
6753    /// is the winner's author, now that the competition is over), another
6754    /// takes a fresh one so the full prompt is sent again. A handover is
6755    /// recorded under `node`, which is what makes the fallback stick for the
6756    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6757    /// is returned, so an exhausted chain reads exactly like a single failed
6758    /// fixer: one quota loss, the same wording, the same refund. An earlier
6759    /// attempt's edits are left in the tree and judged, with the final
6760    /// attempt's, by what git says afterwards.
6761    ///
6762    /// Returns the job that produced the outcome, for `continue_fix_report`.
6763    async fn ask_fixer(
6764        &mut self,
6765        winner: &Candidate,
6766        node: &'static str,
6767        round: Option<usize>,
6768        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6769    ) -> (SeatJob, SeatState, AgentOutcome) {
6770        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6771        self.ask_fixer_among(attempts, node, round, None, build)
6772            .await
6773    }
6774
6775    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6776    /// worktree and a commit: before the next agent is asked after one that
6777    /// did not satisfy the call, the worktree is put back on that commit, so
6778    /// a failed agent's partial edits are not carried into the next one.
6779    async fn ask_fixer_among(
6780        &mut self,
6781        attempts: Vec<(AgentSpec, String)>,
6782        node: &'static str,
6783        round: Option<usize>,
6784        restore: Option<(&Path, &str)>,
6785        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6786    ) -> (SeatJob, SeatState, AgentOutcome) {
6787        let run_id = self.state.id.clone();
6788        let prompts = self.state.config.prompts.clone();
6789        let cache = self.state.config.cache_dir();
6790        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6791        let mut last = None;
6792        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6793            let seat = self.seat(&key, &spec.id);
6794            let mut job = build(spec.clone(), seat);
6795            if i > 0 {
6796                job.stem = format!("{}-{}", job.stem, spec.id);
6797            }
6798            let ctx = WaveCtx {
6799                carry_seats: false,
6800                pauses: Some((&self.pause, &self.interrupt)),
6801                run: &run_id,
6802                node,
6803                prompts: &prompts,
6804                cache: cache.as_deref(),
6805                round,
6806            };
6807            let (seat, out) =
6808                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6809            // A resource block never advances the chain: the chosen agent
6810            // was not asked, so it must be the one asked again on resume.
6811            let advances = match &out {
6812                AgentOutcome::Ok(o) => agent::output_advances(o),
6813                AgentOutcome::ResourceBlocked(_) => false,
6814                _ => true,
6815            };
6816            if let Some(next) = ids.get(i + 1)
6817                && advances
6818            {
6819                self.state.seats.insert(seat.key.clone(), seat.clone());
6820                let class =
6821                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6822                let reason = match &out {
6823                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6824                    other => fail_reason(other),
6825                };
6826                // A failed agent's edits must not reach the next one, and a
6827                // restore that did not work ends the chain: the call fails
6828                // and the caller's own (confirmed) restore decides.
6829                if let Some((wt, rev)) = restore
6830                    && let Err(e) = self.graft_restore(wt, rev).await
6831                {
6832                    let why =
6833                        format!("could not restore the worktree before the next fixer: {e:#}");
6834                    tracing::warn!("{why}");
6835                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6836                        AgentOutcome::ResourceBlocked(why)
6837                    } else {
6838                        AgentOutcome::Failed(why)
6839                    };
6840                    last = Some((job, seat, out));
6841                    break;
6842                }
6843                record_handover(
6844                    &mut self.state,
6845                    node,
6846                    &seat.key,
6847                    &spec.id,
6848                    next,
6849                    &class,
6850                    &reason,
6851                );
6852                continue;
6853            }
6854            last = Some((job, seat, out));
6855            break;
6856        }
6857        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6858        if let AgentOutcome::Quota(output) = &out
6859            && node != "graft"
6860        {
6861            // Graft is best effort: its caller records the loss and restores
6862            // the pre-graft tree, then proceeds to review and gate.
6863            // Every fallback was tried. Machine outages cannot be repaired by
6864            // spending a fix round; use each caller's resumable resource path.
6865            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6866            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6867            self.state.quota.push(loss);
6868            self.state.seats.insert(seat.key.clone(), seat.clone());
6869            self.state.status = RunStatus::Blocked;
6870            return (
6871                job,
6872                seat,
6873                AgentOutcome::ResourceBlocked(
6874                    "fixer unavailable; restore agent availability and resume".into(),
6875                ),
6876            );
6877        }
6878        (job, seat, out)
6879    }
6880
6881    /// The one repair a verification-only failure earns (see
6882    /// [`crate::recovery`]), before the first review round of a run that took
6883    /// a branch over.
6884    ///
6885    /// Saved as started *before* the fixer is called, so a resume never
6886    /// repairs twice; a start found on re-entry is rolled back to the commit
6887    /// it began from. A failed or unusable fixer leaves the tree as it was and
6888    /// the review goes on over it, so the first round's own verification
6889    /// reports the failure again. The failure is never carried into a round as
6890    /// a result: verification is re-run on whatever commit this leaves.
6891    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6892        use crate::recovery::{Mode, Repair};
6893        if !self.state.reviews.is_empty() {
6894            return Ok(());
6895        }
6896        let Some(mut rec) = self.state.recovery.clone() else {
6897            return Ok(());
6898        };
6899        if rec.mode != Mode::VerifyRepair {
6900            return Ok(());
6901        }
6902        let wt = winner.worktree.clone();
6903        let mut head = git::rev_parse(&wt, "HEAD").await?;
6904        match rec.repair.clone() {
6905            Repair::Done { .. } => return Ok(()),
6906            Repair::WaitingForRestore { before } => {
6907                self.graft_restore(&wt, &before).await?;
6908                head = before;
6909                rec.repair = Repair::NotStarted;
6910                self.state.recovery = Some(rec.clone());
6911                self.state.save()?;
6912            }
6913            Repair::Started { before } => {
6914                // Interrupted: the unfinished edits are not trusted.
6915                // An unconfirmed rollback blocks the run with `Started` kept,
6916                // so a resume retries it; the tree is never reviewed as is.
6917                if let Err(e) = self.graft_restore(&wt, &before).await {
6918                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6919                        return Err(e);
6920                    }
6921                    return self
6922                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6923                        .await;
6924                }
6925                rec.repair = Repair::Done {
6926                    before: before.clone(),
6927                    after: before,
6928                    committed: false,
6929                    failed: Some("interrupted; rolled back".to_owned()),
6930                };
6931                self.state
6932                    .event("recovery", "the repair was interrupted and rolled back");
6933                self.state.recovery = Some(rec);
6934                self.state.save()?;
6935                return Ok(());
6936            }
6937            Repair::NotStarted => {}
6938        }
6939        if crate::recovery::recheck(&mut rec, &head) {
6940            self.state.event("recovery", rec.summary());
6941            self.state.recovery = Some(rec);
6942            self.state.save()?;
6943            return Ok(());
6944        }
6945        rec.repair = Repair::Started {
6946            before: head.clone(),
6947        };
6948        self.state.recovery = Some(rec.clone());
6949        self.state.save()?;
6950
6951        let language = self.state.config.graph.language.clone();
6952        let sessions = self.state.config.graph.sessions;
6953        let artifacts = agent::artifacts_dir(&self.state.dir());
6954        let fix_prompt = prompt::recovery_fix(
6955            &self.state.instruction,
6956            &rec,
6957            self.state.config.graph.review_rounds,
6958            &language,
6959        );
6960        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6961        self.state.event(
6962            "recovery",
6963            format!(
6964                "repair: the fixer starts from the failed verification on {}",
6965                short(&head)
6966            ),
6967        );
6968        let (_job, seat, out) = self
6969            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6970                prompt: fix_prompt.clone(),
6971                spec,
6972                seat,
6973                cwd: wt.clone(),
6974                timeout,
6975                allow_write: true,
6976                sessions,
6977                artifacts: artifacts.clone(),
6978                stem: "fix-recovery".to_owned(),
6979                handover: None,
6980            })
6981            .await;
6982        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6983            rec.repair = Repair::WaitingForRestore { before: head };
6984            self.state.recovery = Some(rec);
6985            return self.stop_for_resource();
6986        }
6987        let failed = match &out {
6988            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6989            AgentOutcome::Quota(o) => {
6990                self.state.quota.push(QuotaLoss::from_report(
6991                    seat.key.clone(),
6992                    "fix",
6993                    o.quota.as_ref(),
6994                ));
6995                Some("rate limited (quota); the fixer could not run".to_owned())
6996            }
6997            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6998            AgentOutcome::Failed(e) => Some(e.clone()),
6999        };
7000        self.state.seats.insert(seat.key.clone(), seat);
7001        let coordination = self.coordinate(&wt).await?;
7002        let failed = match failed {
7003            // The agent's partial edits are not kept.
7004            Some(why) => {
7005                if let Err(e) = git::restore_to(&wt, &head).await {
7006                    return self
7007                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
7008                        .await;
7009                }
7010                Some(why)
7011            }
7012            None => {
7013                if let Ok(r) = git::rescue_commit(
7014                    &wt,
7015                    "magi: recovery repair of the failed verification (uncommitted work)",
7016                )
7017                .await
7018                {
7019                    self.state.note_withheld("fix", &r.withheld);
7020                }
7021                None
7022            }
7023        };
7024        let after = git::rev_parse(&wt, "HEAD").await?;
7025        drop(coordination);
7026        self.state.event(
7027            "recovery",
7028            match &failed {
7029                Some(why) => format!("repair: nothing kept ({why})"),
7030                None if after != head => {
7031                    format!("repair: committed {}", short(&after))
7032                }
7033                None => "repair: the fixer made no commit".to_owned(),
7034            },
7035        );
7036        rec.repair = Repair::Done {
7037            before: head.clone(),
7038            committed: after != head,
7039            after,
7040            failed,
7041        };
7042        self.state.recovery = Some(rec);
7043        self.state.save()?;
7044        Ok(())
7045    }
7046
7047    /// A repair whose rollback could not be confirmed: the run is blocked and
7048    /// resumable, and the unconfirmed state is never reviewed.
7049    async fn stop_recovery(
7050        &mut self,
7051        mut rec: crate::recovery::RecoveryRecord,
7052        head: &str,
7053        why: String,
7054    ) -> Result<()> {
7055        // `Started` stays recorded: a resume retries the restore.
7056        rec.repair = crate::recovery::Repair::Started {
7057            before: head.to_owned(),
7058        };
7059        self.state.recovery = Some(rec);
7060        self.state
7061            .event("recovery", format!("repair could not be undone: {why}"));
7062        self.state.status = RunStatus::Blocked;
7063        self.state.save()?;
7064        bail!("recovery repair could not be rolled back: {why}")
7065    }
7066
7067    async fn review_loop(&mut self) -> Result<()> {
7068        if self.hold_base_sync().await? {
7069            return Ok(());
7070        }
7071        loop {
7072            if self
7073                .state
7074                .base_sync
7075                .as_ref()
7076                .is_some_and(|s| s.conflict.is_some())
7077            {
7078                return Ok(());
7079            }
7080            if self.catchup_repairable() {
7081                let record = self.state.reviews.last().expect("repairable round").clone();
7082                let winner = self
7083                    .state
7084                    .winner()
7085                    .cloned()
7086                    .context("catch-up repair has no winner")?;
7087                if !self.fix_review_round(&winner, record).await? {
7088                    return Ok(());
7089                }
7090            }
7091            self.review_loop_pass().await?;
7092            if !self.catchup_repairable() {
7093                return Ok(());
7094            }
7095        }
7096    }
7097
7098    /// Derive an owed catch-up repair from durable round evidence, including
7099    /// reentry after a crash before the fixer was dispatched.
7100    fn catchup_repairable(&self) -> bool {
7101        self.state.reviews.len() < self.state.config.graph.review_rounds
7102            && self.state.reviews.last().is_some_and(|r| {
7103                !r.clean
7104                    && r.blocking == 0
7105                    && r.fix.is_none()
7106                    && r.e2e_status() == E2eStatus::Failed
7107            })
7108    }
7109
7110    async fn review_loop_pass(&mut self) -> Result<()> {
7111        // A base that would not rebase is a person's decision, not a review
7112        // round: nothing here would change the answer, and reviewers and a
7113        // fixer would be spending real budget on a tree that cannot land
7114        // regardless of what they find.
7115        if self
7116            .state
7117            .base_sync
7118            .as_ref()
7119            .is_some_and(|s| s.conflict.is_some())
7120        {
7121            return Ok(());
7122        }
7123        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
7124        // agent files with `magi task add` name the run that paid for it. The
7125        // prompt overlay is cloned alongside it because the waves borrow it
7126        // while `self` is mutably borrowed by the node's own bookkeeping.
7127        let run_id = self.state.id.clone();
7128        let prompts = self.state.config.prompts.clone();
7129        let Some(winner) = self.state.winner().cloned() else {
7130            return Ok(());
7131        };
7132        let granted_review = self
7133            .state
7134            .repair_grants
7135            .last()
7136            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7137            .cloned();
7138        let fresh_review = granted_review
7139            .as_ref()
7140            .is_some_and(|g| self.state.reviews.len() == g.review_start);
7141        let max_rounds = granted_review
7142            .as_ref()
7143            .map_or(self.state.config.graph.review_rounds, |g| {
7144                g.review_start.saturating_add(1)
7145            });
7146        // A clean round, an exhausted round budget, or a stalled tree (see
7147        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
7148        // they are recorded — recomputed here, not read off `status`, so a
7149        // reentry into a run that already stopped restates the identical
7150        // verdict instead of silently handing back whatever an earlier node
7151        // in this same walk clobbered `status` to (a solo-candidate
7152        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
7153        // below runs an empty range once the budget is spent, and would
7154        // otherwise fall through without touching `status` at all.
7155        if let Some(status) =
7156            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7157        {
7158            // A reentry after a crash between the last round's save and
7159            // `stop_reviewing` reaches the hand-off here, not there.
7160            if status == RunStatus::Gating {
7161                self.record_contested_handoff();
7162            }
7163            self.state.status = status;
7164            self.state.save()?;
7165            return Ok(());
7166        }
7167        self.state.status = RunStatus::Reviewing;
7168        // A last recorded round whose own verification never resolved
7169        // (`ResourceBlocked` — the shared build cache, not the patch) is
7170        // never a concluded round, whatever the round budget says: starting
7171        // a fresh round on top of it would spend a whole new reviewer wave
7172        // re-reading an unchanged patch instead of just retrying the one
7173        // check that actually needs it, and once the budget is spent the
7174        // loop below has nothing left to do at all (its range is empty).
7175        // Retry that check directly instead, exactly the same retry
7176        // `stop_reviewing` already does for its own catch-up case.
7177        if !fresh_review
7178            && self.state.reviews.last().is_some_and(|r| {
7179                r.e2e_status() == E2eStatus::ResourceBlocked
7180                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7181            })
7182        {
7183            let shell = self.state.config.shell();
7184            return self
7185                .stop_unresolved_review(
7186                    "the last round's own verification never resolved",
7187                    &shell,
7188                    &winner.worktree,
7189                )
7190                .await;
7191        }
7192
7193        if granted_review.is_none() {
7194            self.recovery_repair(&winner).await?;
7195        }
7196        if self.resource_halt()? {
7197            return Ok(());
7198        }
7199
7200        let repo = self.state.repo.clone();
7201        let root = self.state.worktree_root();
7202        let language = self.state.config.graph.language.clone();
7203        let sessions = self.state.config.graph.sessions;
7204        let artifacts = self.merge_repair_artifacts();
7205        let base = self.landing_base();
7206        let base_short = short(&base);
7207        let reviewers = self.roles.reviewers.clone();
7208        let shell = self.state.config.shell();
7209
7210        for round in (self.state.reviews.len() + 1)..=max_rounds {
7211            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7212            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7213            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7214            // The prior round's own record, already persisted — never a
7215            // hand-carried variable of just its failing output: that is
7216            // exactly what let a round's e2e result drift out of sync with
7217            // which commit it was actually about (see `SCHEMA`'s doc for
7218            // schema 8). Judged against `head`, the commit reviewers are
7219            // about to look at now, so the summary always reads as "an
7220            // earlier head" here — this round's own patch has not been
7221            // checked yet.
7222            let prev_verification = self
7223                .state
7224                .reviews
7225                .last()
7226                .and_then(|r| r.verification_summary(&head));
7227            // Only the first round of a run that took a branch over: later
7228            // rounds have this run's own history.
7229            let prior_section = match (&self.state.recovery, round) {
7230                (Some(rec), 1) if self.state.reviews.is_empty() => {
7231                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7232                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7233                            Some(crate::recovery::head_of(
7234                                &git::diff(&winner.worktree, reviewed, "HEAD")
7235                                    .await
7236                                    .unwrap_or_default(),
7237                                crate::recovery::SINCE_DIFF_MAX,
7238                            ))
7239                        }
7240                        _ => None,
7241                    };
7242                    Some(prompt::recovery_review(rec, since.as_deref()))
7243                }
7244                _ => None,
7245            };
7246
7247            // Each reviewer gets its own detached checkout of exactly this
7248            // commit: nobody can perturb the winner's tree, and the fixer can
7249            // keep working without racing a reviewer.
7250            let mut jobs = Vec::new();
7251            let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7252                self.coordinate(&repo).await?
7253            } else {
7254                None
7255            };
7256            for (r, spec) in reviewers.iter().cloned().enumerate() {
7257                let wt = root.join(format!("review-{}", r + 1));
7258                // `clean -fdx` in a reset wipes setup's products, so every
7259                // round sets the seat up again.
7260                if wt.exists() {
7261                    crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7262                } else {
7263                    let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7264                        None
7265                    } else {
7266                        self.coordinate(&repo).await?
7267                    };
7268                    git::worktree_add_detached(&repo, &wt, &head).await?;
7269                }
7270                let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7271                    None
7272                } else {
7273                    self.coordinate(&repo).await?
7274                };
7275                self.setup_seat_worktree(&repo, &wt, false).await?;
7276                let seat_key = format!("review-{}", r + 1);
7277                // The seat starts the round on whoever answered it last, not
7278                // on the agent the spec names, so a failure is not re-paid.
7279                let spec = pick_start_spec(
7280                    &self.roles.reviewer_roster,
7281                    spec,
7282                    self.state.seat_history.get(&seat_key),
7283                );
7284                let seat = self.seat(&seat_key, &spec.id);
7285                jobs.push(SeatJob {
7286                    prompt: prompt::review(&prompt::ReviewCtx {
7287                        instruction: &self.state.instruction,
7288                        branch: &winner.branch,
7289                        base_short: &base_short,
7290                        stat: &stat,
7291                        patch: &patch,
7292                        verification: prev_verification.as_ref(),
7293                        prior: prior_section.as_deref(),
7294                        reviewers: reviewers.len(),
7295                        round,
7296                        rounds: max_rounds,
7297                        // A review-only run has no rankings, so nothing
7298                        // competed for this patch and the reviewer is told so.
7299                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7300                        lens: Lens::for_seat(r),
7301                        language: &language,
7302                        github_language: &self.state.config.graph.github_language,
7303                    }),
7304                    spec,
7305                    seat,
7306                    cwd: wt,
7307                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7308                    allow_write: false,
7309                    sessions,
7310                    artifacts: artifacts.clone(),
7311                    stem: format!("review-{round}-{}", r + 1),
7312                    handover: None,
7313                });
7314            }
7315
7316            drop(coordination);
7317            self.state.event(
7318                "review",
7319                format!(
7320                    "round {round}: {} reviewers on {}",
7321                    jobs.len(),
7322                    short(&head)
7323                ),
7324            );
7325            let mut quota_losses = Vec::new();
7326            let review_retries = self.state.config.graph.retries;
7327            let review_cache = self.state.config.cache_dir();
7328            let ctx = WaveCtx {
7329                carry_seats: true,
7330                pauses: Some((&self.pause, &self.interrupt)),
7331                run: &run_id,
7332                node: "review",
7333                prompts: &prompts,
7334                cache: review_cache.as_deref(),
7335                round: Some(round),
7336            };
7337            let results = ask_json_wave::<Review>(
7338                jobs,
7339                Arc::clone(&self.sem),
7340                review_retries,
7341                &self.roles.reviewer_roster,
7342                &ctx,
7343                &mut quota_losses,
7344                &mut self.state,
7345                &|_: &Review| Ok(()),
7346            )
7347            .await;
7348            if self.state.resource_block.is_some() {
7349                return self.stop_for_resource();
7350            }
7351            // Counted before the move below: how many of *this* round's
7352            // reviewer seats were lost to their own rate limit, as opposed to
7353            // a crash, a timeout, or unparsable output — see `round_is_clean`.
7354            let round_quota_missing = quota_losses.len();
7355            self.state.quota.extend(quota_losses);
7356
7357            let mut records = Vec::new();
7358            let mut all_findings = Vec::new();
7359            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7360                let agent_id = seat.agent.clone();
7361                self.state.seats.insert(seat.key.clone(), seat);
7362                let mut record = ReviewRecord {
7363                    reviewer: r + 1,
7364                    agent: agent_id,
7365                    summary: String::new(),
7366                    findings: Vec::new(),
7367                    vote: None,
7368                    failed: None,
7369                    duration_ms: 0,
7370                    // Set for both outcomes: `failed: Some(_)` with
7371                    // `attempts > 0` is a seat every retry still lost, not a
7372                    // recovered one — only `failed: None` with `attempts > 0`
7373                    // reads as "answered after a nudge" (see this field's own
7374                    // doc).
7375                    attempts,
7376                };
7377                match res {
7378                    Ok((review, out)) => {
7379                        // Sanitized here, at the point every other piece of
7380                        // agent prose in this file is (candidate summaries,
7381                        // deliberation turns, vote reasons): a reviewer's own
7382                        // words are the one thing about it that could name
7383                        // it, and reconsideration below broadcasts this same
7384                        // summary and these same findings to every other
7385                        // seat on the panel.
7386                        record.summary =
7387                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
7388                        record.vote = Some(review.vote);
7389                        record.duration_ms = out.duration_ms;
7390                        for (n, mut f) in review.findings.into_iter().enumerate() {
7391                            // ids are magi's, never the agent's: the fixer's
7392                            // adoption report is keyed by them.
7393                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7394                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7395                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7396                            // `file` is agent-supplied prose too, never
7397                            // checked against the real tree — the same
7398                            // exposure `title`/`detail` above have, just in
7399                            // a field easy to forget because it looks like a
7400                            // path rather than free text.
7401                            f.file = f
7402                                .file
7403                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7404                            all_findings.push(f.clone());
7405                            record.findings.push(f);
7406                        }
7407                        self.state.event(
7408                            "review",
7409                            format!(
7410                                "round {round}: reviewer {} voted {} with {} finding(s)",
7411                                r + 1,
7412                                review.vote.label(),
7413                                record.findings.len()
7414                            ),
7415                        );
7416                    }
7417                    Err(e) => {
7418                        record.failed = Some(e.to_string());
7419                        self.state.event(
7420                            "review",
7421                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7422                        );
7423                    }
7424                }
7425                records.push(record);
7426            }
7427
7428            // Tally the round's votes and, if they split, spend the one
7429            // round of reconsideration the split -> deliberate -> revote
7430            // shape `judge`/`vote` use for the panel, sized down to what a
7431            // read-only review round can afford: one round, and a revote
7432            // rather than an argument, because the panel already wrote its
7433            // reasoning down as findings the first time around.
7434            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7435            let vote_split =
7436                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7437            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7438            if vote_split {
7439                self.state.event(
7440                    "review",
7441                    format!(
7442                        "round {round}: votes split ({}) — one round of reconsideration",
7443                        initial_votes
7444                            .iter()
7445                            .map(|v| v.label())
7446                            .collect::<Vec<_>>()
7447                            .join(", ")
7448                    ),
7449                );
7450                // Seats read every seat's findings and votes, still numbered
7451                // and never named — the same anonymity `review` itself keeps.
7452                let panel: Vec<ReviewSeatReport<'_>> = records
7453                    .iter()
7454                    .filter_map(|r| {
7455                        r.vote.map(|vote| ReviewSeatReport {
7456                            reviewer: r.reviewer,
7457                            vote,
7458                            summary: &r.summary,
7459                            findings: &r.findings,
7460                        })
7461                    })
7462                    .collect();
7463
7464                let mut jobs = Vec::new();
7465                let mut seats_at = Vec::new();
7466                for (r, spec) in reviewers.iter().cloned().enumerate() {
7467                    // A seat with no initial vote has nothing to reconsider
7468                    // from and stays absent, the same as it stayed absent
7469                    // from `panel` above.
7470                    if records[r].vote.is_none() {
7471                        continue;
7472                    }
7473                    let wt = root.join(format!("review-{}", r + 1));
7474                    let seat_key = format!("review-{}", r + 1);
7475                    let spec = self.occupant(&seat_key, spec);
7476                    let seat = self.seat(&seat_key, &spec.id);
7477                    // A seat with no live session has already forgotten the
7478                    // initial review's prompt — restate the patch it is
7479                    // voting on, the same as `deliberate`/`vote` do for a
7480                    // judge in the same position.
7481                    // The panel already carries this seat's own review and
7482                    // vote, so restating the patch makes the prompt whole for
7483                    // a seat handed to another agent.
7484                    let build = |with_patch: bool| {
7485                        prompt::review_reconsider(&ReviewReconsiderCtx {
7486                            instruction: &self.state.instruction,
7487                            reviewer: r + 1,
7488                            lens: Lens::for_seat(r),
7489                            panel: &panel,
7490                            patch: with_patch.then_some(ReviewPatch {
7491                                branch: &winner.branch,
7492                                base_short: &base_short,
7493                                stat: &stat,
7494                                patch: &patch,
7495                            }),
7496                            round,
7497                            rounds: max_rounds,
7498                            language: &language,
7499                        })
7500                    };
7501                    let full = build(true);
7502                    let prompt = if has_context(&spec, &seat, sessions) {
7503                        build(false)
7504                    } else {
7505                        full.clone()
7506                    };
7507                    jobs.push(SeatJob {
7508                        prompt,
7509                        spec,
7510                        seat,
7511                        cwd: wt,
7512                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7513                        allow_write: false,
7514                        sessions,
7515                        artifacts: artifacts.clone(),
7516                        stem: format!("review-{round}-reconsider-{}", r + 1),
7517                        handover: Some(full),
7518                    });
7519                    seats_at.push(r);
7520                }
7521
7522                let mut recon_quota_losses = Vec::new();
7523                let recon_cache = self.state.config.cache_dir();
7524                let recon_ctx = WaveCtx {
7525                    carry_seats: true,
7526                    pauses: Some((&self.pause, &self.interrupt)),
7527                    run: &run_id,
7528                    node: "review",
7529                    prompts: &prompts,
7530                    cache: recon_cache.as_deref(),
7531                    round: Some(round),
7532                };
7533                let recon_results = ask_json_wave::<ReviewRevote>(
7534                    jobs,
7535                    Arc::clone(&self.sem),
7536                    review_retries,
7537                    &self.roles.reviewer_roster,
7538                    &recon_ctx,
7539                    &mut recon_quota_losses,
7540                    &mut self.state,
7541                    &|_: &ReviewRevote| Ok(()),
7542                )
7543                .await;
7544                if self.state.resource_block.is_some() {
7545                    return self.stop_for_resource();
7546                }
7547                self.state.quota.extend(recon_quota_losses);
7548
7549                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7550                    let agent_id = seat.agent.clone();
7551                    self.state.seats.insert(seat.key.clone(), seat);
7552                    let mut rec = ReviewRevoteRecord {
7553                        reviewer: r + 1,
7554                        agent: agent_id,
7555                        vote: None,
7556                        reason: String::new(),
7557                        failed: None,
7558                    };
7559                    match res {
7560                        Ok((rv, _)) => {
7561                            rec.vote = Some(rv.vote);
7562                            rec.reason =
7563                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7564                            self.state.event(
7565                                "review",
7566                                format!(
7567                                    "round {round}: reviewer {} revoted {}",
7568                                    r + 1,
7569                                    rv.vote.label()
7570                                ),
7571                            );
7572                        }
7573                        Err(e) => {
7574                            rec.failed = Some(e.to_string());
7575                            self.state.event(
7576                                "review",
7577                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7578                            );
7579                        }
7580                    }
7581                    reconsideration.push(rec);
7582                }
7583            } else if initial_votes.len() > 1 {
7584                self.state.event(
7585                    "review",
7586                    format!(
7587                        "round {round}: votes agreed ({}) — no reconsideration",
7588                        initial_votes[0].label()
7589                    ),
7590                );
7591            }
7592
7593            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7594            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7595            // A round that already has a blocking finding and a round left to
7596            // try is going back to the fixer no matter what `verify.e2e`
7597            // says, so running it first only spends the loop's slowest step
7598            // (minutes, for a Rust repo's full test suite) on a head about
7599            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7600            // runs once a round has no blocking findings left (see
7601            // `round_is_clean`, which a deferred — empty — `e2e` can never
7602            // satisfy since `blocking` is nonzero whenever this branch is
7603            // taken), and `stop_reviewing` forces a real run before it will
7604            // ever read a deferred round as green.
7605            let defer_e2e =
7606                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7607            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7608                let reason =
7609                    format!("{blocking} blocking finding(s) already required a fix this round");
7610                self.state.event(
7611                    "verify",
7612                    format!(
7613                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7614                         {}); it will run once a round has none left",
7615                        short(&head)
7616                    ),
7617                );
7618                (Vec::new(), false, true, Some(reason))
7619            } else {
7620                let e2e_commands = self.state.config.verify.e2e.clone();
7621                let cache_dir = self.state.config.cache_dir();
7622                let context = format!("round {round}");
7623                let wait = self.repository_wait();
7624                let (e2e, verify_retried) = with_cache_lease(
7625                    &mut self.state,
7626                    cache_dir.as_deref(),
7627                    "e2e",
7628                    "e2e",
7629                    &winner.worktree,
7630                    &head,
7631                    verify_timeout,
7632                    &context,
7633                    &wait,
7634                    |state, budget| {
7635                        let shell = shell.clone();
7636                        let e2e_commands = e2e_commands.clone();
7637                        let worktree = winner.worktree.clone();
7638                        let context = context.clone();
7639                        async move {
7640                            run_e2e_with_retry(
7641                                state,
7642                                &shell,
7643                                &e2e_commands,
7644                                &worktree,
7645                                budget,
7646                                &context,
7647                            )
7648                            .await
7649                        }
7650                    },
7651                )
7652                .await?;
7653                (e2e, verify_retried, false, None)
7654            };
7655
7656            let expected = records.len();
7657            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7658            let incomplete = answered < expected;
7659            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7660            let policy = self.state.config.graph.incomplete_review;
7661            let clean = round_is_clean(
7662                blocking,
7663                e2e_ok,
7664                answered,
7665                expected,
7666                round_quota_missing,
7667                policy,
7668            );
7669
7670            let mut round_record = ReviewRound {
7671                round,
7672                head: head.clone(),
7673                verified_head: None,
7674                verified_at: None,
7675                reviews: records,
7676                e2e,
7677                verify_retried,
7678                e2e_deferred,
7679                e2e_defer_reason,
7680                fix: None,
7681                blocking,
7682                answered,
7683                expected,
7684                clean,
7685                progressed: false,
7686                vote_split,
7687                reconsideration,
7688                verdict: None,
7689            };
7690            // The final vote per seat is its revote where reconsideration
7691            // ran and answered, its initial vote otherwise — the same
7692            // fallback `tally` uses for a judge whose private vote failed.
7693            round_record.verdict = ReviewVote::worst(
7694                round_record
7695                    .final_votes()
7696                    .into_iter()
7697                    .map(|(_, _, vote)| vote),
7698            );
7699            // Which commit and when magi actually attempted to check —
7700            // known the moment a command was dispatched against `head`,
7701            // whether or not it finished: a resource-blocked attempt still
7702            // targeted a specific commit at a specific time, and leaving
7703            // that unrecorded is exactly what made `verification_summary`
7704            // report a fresh attempt as "commit unknown ... recorded before
7705            // this was tracked", indistinguishable from a genuinely old,
7706            // untracked record. Only a deferred or unconfigured round never
7707            // ran at all and has nothing to record — see
7708            // `ReviewRound::verified_head`'s own doc.
7709            if !matches!(
7710                round_record.e2e_status(),
7711                E2eStatus::Deferred | E2eStatus::NotConfigured
7712            ) {
7713                round_record.verified_head = Some(head.clone());
7714                round_record.verified_at = Some(Timestamp::now());
7715            }
7716
7717            if incomplete {
7718                let missing: Vec<String> = round_record
7719                    .reviews
7720                    .iter()
7721                    .filter(|r| r.failed.is_some())
7722                    .map(|r| format!("review-{}", r.reviewer))
7723                    .collect();
7724                self.state.event(
7725                    "review",
7726                    format!(
7727                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7728                        missing.join(", ")
7729                    ),
7730                );
7731            }
7732
7733            if clean {
7734                self.state.event(
7735                    "review",
7736                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7737                        format!(
7738                            "round {round}: clean (warn policy, incomplete panel) — no \
7739                             blocking findings from the seats that answered, verification green"
7740                        )
7741                    } else if incomplete {
7742                        format!(
7743                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7744                             quorum) — no blocking findings from the seats that answered, \
7745                             verification green",
7746                            expected - answered
7747                        )
7748                    } else {
7749                        format!("round {round}: clean — no blocking findings, verification green")
7750                    },
7751                );
7752                self.state.reviews.push(round_record);
7753                self.state.status = RunStatus::Gating;
7754                self.state.save()?;
7755                return Ok(());
7756            }
7757
7758            // Nothing was raised and verification passed, but not every seat
7759            // answered and `round_is_clean` still refused to call it clean —
7760            // either a seat is missing for a reason other than its own quota
7761            // (a crash, a timeout, unparsable output — worth another try), or
7762            // every seat that could have answered lost its quota and nobody
7763            // is left to decide on: re-review rather than send the fixer
7764            // after a round with nothing to fix.
7765            if incomplete && blocking == 0 && e2e_ok {
7766                self.state.reviews.push(round_record);
7767                self.state.save()?;
7768                if round == max_rounds {
7769                    self.state.status = RunStatus::Blocked;
7770                    self.state.event(
7771                        "review",
7772                        format!(
7773                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7774                             refusing to call it clean",
7775                            expected - answered
7776                        ),
7777                    );
7778                    return Ok(());
7779                }
7780                continue;
7781            }
7782
7783            // Nothing for the fixer to act on (`blocking == 0`) and the only
7784            // reason this round is not clean is that magi itself never got
7785            // a command to run — the shared build cache, not the patch (see
7786            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7787            // the fixer would invite a change to appease contention that has
7788            // nothing to do with the diff, and would leave this attempt
7789            // sitting in the next round's prompt as if it were about an
7790            // earlier, superseded commit rather than what it actually is:
7791            // the same head, still waiting to be checked. Wait for it the
7792            // same way the final round's own contention is already handled,
7793            // whatever round this happens to be.
7794            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7795                self.state.reviews.push(round_record);
7796                return self
7797                    .stop_unresolved_review(
7798                        "the round's own verification could not run",
7799                        &shell,
7800                        &winner.worktree,
7801                    )
7802                    .await;
7803            }
7804
7805            if round == max_rounds {
7806                self.state.reviews.push(round_record);
7807                return self
7808                    .stop_reviewing(
7809                        &format!(
7810                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7811                        ),
7812                        &shell,
7813                        &winner.worktree,
7814                    )
7815                    .await;
7816            }
7817
7818            if !self.fix_review_round(&winner, round_record).await? {
7819                return Ok(());
7820            }
7821        }
7822        Ok(())
7823    }
7824
7825    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7826    /// False means admission or continuation policy stopped the review loop.
7827    async fn fix_review_round(
7828        &mut self,
7829        winner: &Candidate,
7830        mut round_record: ReviewRound,
7831    ) -> Result<bool> {
7832        let round = round_record.round;
7833        let shell = self.state.config.shell();
7834        let step = self.run_fix_round(winner, &mut round_record).await?;
7835        if matches!(step, FixStep::Resource) {
7836            self.stop_for_resource()?;
7837            return Ok(false);
7838        }
7839        if self.state.reviews.len() < round {
7840            self.state.reviews.push(round_record);
7841        } else {
7842            self.state.reviews[round - 1] = round_record;
7843        }
7844        self.state.save()?;
7845
7846        // The fixer's own report never came back this round, even after
7847        // `continue_fix_report`'s own budget was spent on it — not an
7848        // ordinary "no report" (dropped stream, quota, plain failure),
7849        // which already reads that way and is left to the existing round
7850        // budget. Stopping here, rather than opening another round, is
7851        // what keeps a next reviewer/fixer wave from ever being
7852        // dispatched onto `winner.worktree` while whatever the seat's
7853        // last call may still have running there is unaccounted for: no
7854        // process liveness check exists (and none is being added — see
7855        // AGENTS.md/this task's own scope), so the only way to honour
7856        // "nothing starts before a valid report returns" is to not start
7857        // anything further on this worktree from this run at all.
7858        if matches!(step, FixStep::Recorded { report_lost: true }) {
7859            self.stop_reviewing(
7860                "the fixer's adoption report never came back, even after resuming its \
7861                         own seat; refusing to start another round against the same worktree \
7862                         while that is unresolved",
7863                &shell,
7864                &winner.worktree,
7865            )
7866            .await?;
7867            return Ok(false);
7868        }
7869
7870        let streak = self
7871            .state
7872            .reviews
7873            .iter()
7874            .rev()
7875            .take_while(|r| !r.progressed)
7876            .count();
7877        if streak >= STAGNANT_LIMIT {
7878            self.stop_reviewing(
7879                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7880                &shell,
7881                &winner.worktree,
7882            )
7883            .await?;
7884            return Ok(false);
7885        }
7886        Ok(true)
7887    }
7888
7889    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7890    async fn run_fix_round(
7891        &mut self,
7892        winner: &Candidate,
7893        round_record: &mut ReviewRound,
7894    ) -> Result<FixStep> {
7895        let round = round_record.round;
7896        let max_rounds = self.state.config.graph.review_rounds;
7897        let run_id = self.state.id.clone();
7898        let prompts = self.state.config.prompts.clone();
7899        let language = self.state.config.graph.language.clone();
7900        let sessions = self.state.config.graph.sessions;
7901        let artifacts = self.merge_repair_artifacts();
7902        let base = self.landing_base();
7903        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7904        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7905        let this_round_verification = round_record.verification_summary(&head);
7906        let all_findings: Vec<_> = round_record
7907            .reviews
7908            .iter()
7909            .flat_map(|r| r.findings.iter().cloned())
7910            .collect();
7911        // Fix. The winner's own implementer seat continues its conversation:
7912        // the competition is over, so context is pure benefit now.
7913        let blocking_findings: Vec<_> = all_findings
7914            .iter()
7915            .filter(|f| f.severity.blocks())
7916            .cloned()
7917            .collect();
7918        let fix_prompt = prompt::fix_in(
7919            &self.state.instruction,
7920            &blocking_findings,
7921            this_round_verification.as_ref(),
7922            round,
7923            max_rounds,
7924            &language,
7925            &self.state.config.graph.github_language,
7926        );
7927        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7928        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7929        let (job, seat, out) = self
7930            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7931                prompt: fix_prompt.clone(),
7932                spec,
7933                seat,
7934                cwd: winner.worktree.clone(),
7935                timeout,
7936                allow_write: true,
7937                sessions,
7938                artifacts: artifacts.clone(),
7939                stem: format!("fix-{round}"),
7940                handover: None,
7941            })
7942            .await;
7943        // Not asked: the round is not recorded (so a resume repeats it),
7944        // no fixer was handed over to, and the seat is left as it was.
7945        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7946            return Ok(FixStep::Resource);
7947        }
7948        let agent_id = seat.agent.clone();
7949
7950        let mut fix = FixRecord {
7951            agent: agent_id,
7952            addressed: Vec::new(),
7953            rejected: Vec::new(),
7954            notes: String::new(),
7955            committed: false,
7956            failed: None,
7957            duration_ms: 0,
7958            continuation: None,
7959        };
7960        let mut continuation = ContinuationRecord::not_needed();
7961        let mut final_seat = seat.clone();
7962        match out {
7963            AgentOutcome::Ok(o) => {
7964                fix.duration_ms = o.duration_ms;
7965                let parsed = verdict::extract_json::<FixReport>(&o.text);
7966                // A parsed report standing next to a command this same
7967                // reply's own CLI never confirmed the exit status of is
7968                // not a resolved answer — the identical `CommandEvidence`
7969                // `state.jobs` renders, read here instead of only on
7970                // display, per the completion judgment and the shown
7971                // record needing to agree.
7972                let incomplete_reason = match &parsed {
7973                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7974                        "the reply parsed, but it reported a command whose own CLI never \
7975                             confirmed an exit status"
7976                            .to_owned(),
7977                    ),
7978                    Ok(_) => None,
7979                    Err(e) => Some(e.to_string()),
7980                };
7981                match incomplete_reason {
7982                    None => {
7983                        let report = parsed.expect("checked Ok above");
7984                        fix.addressed = report.addressed;
7985                        fix.rejected = report.rejected;
7986                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7987                    }
7988                    Some(reason) => {
7989                        let (resumed_seat, resolved, failure, cont) = self
7990                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7991                            .await;
7992                        fix.duration_ms += cont.cumulative_wait_ms;
7993                        continuation = cont;
7994                        final_seat = resumed_seat;
7995                        match resolved {
7996                            Some(report) => {
7997                                fix.addressed = report.addressed;
7998                                fix.rejected = report.rejected;
7999                                fix.notes =
8000                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
8001                            }
8002                            None => fix.failed = failure,
8003                        }
8004                    }
8005                }
8006            }
8007            // The CLI's raw error JSON is not a fix report to parse.
8008            AgentOutcome::Dropped(o) => {
8009                fix.duration_ms = o.duration_ms;
8010                let why = o
8011                    .dropped
8012                    .as_ref()
8013                    .map(|d| d.why.as_str())
8014                    .unwrap_or("the CLI ended the stream without delivering its answer");
8015                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8016            }
8017            AgentOutcome::Quota(o) => {
8018                self.state.quota.push(QuotaLoss::from_report(
8019                    final_seat.key.clone(),
8020                    "fix",
8021                    o.quota.as_ref(),
8022                ));
8023                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8024            }
8025            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8026        }
8027        if self.state.resource_block.is_some() {
8028            self.state.seats.insert(final_seat.key.clone(), final_seat);
8029            return Ok(FixStep::Resource);
8030        }
8031        fix.continuation = Some(continuation);
8032        self.state.seats.insert(final_seat.key.clone(), final_seat);
8033        let coordination = self.coordinate(&winner.worktree).await?;
8034        if let Ok(r) = git::rescue_commit(
8035            &winner.worktree,
8036            &format!("magi: review round {round} fixes (uncommitted work)"),
8037        )
8038        .await
8039        {
8040            self.state.note_withheld("fix", &r.withheld);
8041        }
8042        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8043        fix.committed = after != before;
8044        // Judged by what `git` says moved against base, never by the
8045        // fixer's own `addressed`/`rejected` count — see
8046        // `ReviewRound::progressed`. Propagated with `?`, the same as the
8047        // `patch` snapshot above: swallowing this error would default
8048        // `diff_after` to empty, which almost always differs from a
8049        // non-empty `patch` and reads as "progressed" — exactly backwards
8050        // for a `git` failure the stagnation check cannot see through.
8051        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8052        drop(coordination);
8053        let progressed = diff_after != patch;
8054        let commit_note = if fix.committed {
8055            "committed"
8056        } else {
8057            "NO new commit"
8058        };
8059        let tree_note = if progressed {
8060            "changed vs base"
8061        } else {
8062            "unchanged vs base"
8063        };
8064        self.state.event(
8065            "fix",
8066            match &fix.failed {
8067                // Distinct on purpose from "0 addressed, 0 rejected": the
8068                // fixer's own diff still landed (blocking counts do keep
8069                // falling round over round), only its adoption report did
8070                // not come back, so this must never read like every
8071                // finding was reviewed and declined.
8072                Some(reason) => {
8073                    format!(
8074                        "round {round}: fixer's adoption report was lost ({reason}); \
8075                             {commit_note}, tree {tree_note}"
8076                    )
8077                }
8078                None => format!(
8079                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8080                         {tree_note}{}",
8081                    fix.addressed.len(),
8082                    fix.rejected.len(),
8083                    if continuation.outcome == ContinuationOutcome::Resumed {
8084                        format!(
8085                            " (adoption report recovered after {} continuation(s))",
8086                            continuation.attempts
8087                        )
8088                    } else {
8089                        String::new()
8090                    },
8091                ),
8092            },
8093        );
8094        round_record.fix = Some(fix);
8095        round_record.progressed = progressed;
8096        Ok(FixStep::Recorded {
8097            report_lost: matches!(
8098                continuation.outcome,
8099                ContinuationOutcome::Exhausted
8100                    | ContinuationOutcome::QuotaLost
8101                    | ContinuationOutcome::NoSession
8102            ),
8103        })
8104    }
8105
8106    /// Decide, from the last recorded round's own verification, whether
8107    /// stopping the review loop is a hand-off or a genuine block.
8108    ///
8109    /// Called once the loop has given up trying — the round budget is spent,
8110    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
8111    /// findings still open, never while a round is still clean or the
8112    /// incomplete-panel case handled inline above. Gate and e2e are facts
8113    /// about the tree; a lingering review finding is an opinion, and this
8114    /// workload's own `magi stats` puts reviewer precision low enough
8115    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
8116    /// must not by itself stand between a green, verified change and the
8117    /// human who decides what to do with it. A red e2e is not an opinion, so
8118    /// that case still blocks, with the failing command and a tail of its
8119    /// output recorded here rather than left in `run.json` for someone to go
8120    /// find.
8121    ///
8122    /// A round that deferred its own e2e (see [`Config::graph`]'s
8123    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
8124    /// only because nothing ran, and treating an empty list as a passing one
8125    /// here is exactly the "deferred painted green" bug this function exists
8126    /// to not have. When the last round's own verification never resolved —
8127    /// deferred on purpose, or a real attempt the shared build cache blocked
8128    /// — this makes (or retries) the real run, on the actual worktree this
8129    /// loop is about to stop touching, before deciding anything. A
8130    /// resource-blocked attempt is likewise never read as either green or
8131    /// red: it is evidence about the machine, not the patch (see
8132    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
8133    /// blocked cache leaves this call without deciding rather than guessing
8134    /// — the caller retries on a later reentry.
8135    /// Record, once, that the review loop handed off over a blocking finding
8136    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
8137    /// `land` asks the owner even with `land_approval` off. Called from every
8138    /// path that concludes `Gating`; a reentry keeps the first record.
8139    fn record_contested_handoff(&mut self) {
8140        if self.state.contested_handoff.is_some() {
8141            return;
8142        }
8143        let Some(contested) = self
8144            .state
8145            .reviews
8146            .last()
8147            .and_then(ReviewRound::contested_handoff)
8148        else {
8149            return;
8150        };
8151        self.state.event(
8152            "review",
8153            format!(
8154                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8155                 wait for the owner's approval",
8156                contested.findings.len(),
8157                contested.rejecters.len()
8158            ),
8159        );
8160        self.state.contested_handoff = Some(contested);
8161    }
8162
8163    async fn stop_unresolved_review(
8164        &mut self,
8165        why: &str,
8166        shell: &[String],
8167        worktree: &Path,
8168    ) -> Result<()> {
8169        self.stop_reviewing_inner(why, shell, worktree, true).await
8170    }
8171
8172    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8173        self.stop_reviewing_inner(why, shell, worktree, false).await
8174    }
8175
8176    async fn stop_reviewing_inner(
8177        &mut self,
8178        why: &str,
8179        shell: &[String],
8180        worktree: &Path,
8181        repairable: bool,
8182    ) -> Result<()> {
8183        let round_idx = self.state.reviews.len() - 1;
8184        // A deferred round and a resource-blocked one are the same shape
8185        // here: neither has a real result yet, and both get one more
8186        // attempt. Read off `e2e_status` — the single source for this —
8187        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
8188        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
8189        // still retries instead of being read as a settled result the
8190        // instant it stops being empty.
8191        let needs_catchup_run = matches!(
8192            self.state.reviews[round_idx].e2e_status(),
8193            E2eStatus::Deferred | E2eStatus::ResourceBlocked
8194        );
8195        if needs_catchup_run {
8196            let round = self.state.reviews[round_idx].round;
8197            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8198            let commands = self.state.config.verify.e2e.clone();
8199            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8200            let cache_dir = self.state.config.cache_dir();
8201            let context = format!(
8202                "round {round}: verification unresolved, catching up before the final decision"
8203            );
8204            let wait = self.repository_wait();
8205            let (outcomes, verify_retried) = with_cache_lease(
8206                &mut self.state,
8207                cache_dir.as_deref(),
8208                "e2e",
8209                "e2e",
8210                worktree,
8211                &attempted_head,
8212                timeout,
8213                &context,
8214                &wait,
8215                |state, budget| {
8216                    let shell = shell.to_vec();
8217                    let commands = commands.clone();
8218                    let context = context.clone();
8219                    async move {
8220                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8221                            .await
8222                    }
8223                },
8224            )
8225            .await?;
8226            if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8227                self.state.record_base_progress(&attempted_head);
8228            }
8229            let last = &mut self.state.reviews[round_idx];
8230            last.e2e = outcomes;
8231            last.verify_retried = verify_retried;
8232            // Always the commit and time this attempt actually targeted,
8233            // whether or not it happens to equal the reviewed `head` and
8234            // whether or not a command finished — see
8235            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
8236            // attempt is recorded too, so a later reader sees "attempted
8237            // again at T2" rather than silence.
8238            last.verified_head = Some(attempted_head);
8239            last.verified_at = Some(Timestamp::now());
8240            if verify_inconclusive(&last.e2e) {
8241                // Still not a real result: `e2e_deferred` is left exactly
8242                // as it was, so `needs_catchup_run` above reads
8243                // `ResourceBlocked` (via `e2e_status`, which checks
8244                // `resource_blocked` before `e2e_deferred`) and retries
8245                // again on the next reentry, rather than recording
8246                // contention as a red e2e and blocking the run on it.
8247                self.state.save()?;
8248                return Ok(());
8249            }
8250            last.e2e_deferred = false;
8251            // A complete panel may have been saved as unclean solely because
8252            // verification could not run. Refresh that verdict with the new
8253            // outcomes before a granted repair consumes the round's result.
8254            // Incomplete panels retain their original policy/quota decision.
8255            if last.answered == last.expected {
8256                last.clean = round_is_clean(
8257                    last.blocking,
8258                    last.e2e.iter().all(CommandOutcome::ok),
8259                    last.answered,
8260                    last.expected,
8261                    0,
8262                    self.state.config.graph.incomplete_review,
8263                );
8264            }
8265        }
8266        let last = &self.state.reviews[round_idx];
8267        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8268
8269        match last.e2e_status() {
8270            E2eStatus::Failed => {
8271                let red: Vec<String> = last
8272                    .e2e
8273                    .iter()
8274                    .filter(|o| !o.ok())
8275                    .map(|o| {
8276                        format!(
8277                            "`{}` -> {:?}\n{}",
8278                            o.command,
8279                            o.code,
8280                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8281                        )
8282                    })
8283                    .collect();
8284                let can_repair = last.blocking == 0 && last.fix.is_none();
8285                self.state.reviews[round_idx].clean = false;
8286                self.state
8287                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8288                self.state.status = RunStatus::Blocked;
8289                if repairable
8290                    && can_repair
8291                    && self.state.reviews.len() < self.state.config.graph.review_rounds
8292                {
8293                    self.state.status = RunStatus::Reviewing;
8294                }
8295            }
8296            // `needs_catchup_run` above already retried once this call; if
8297            // it is still blocked, this is magi's own admission it could
8298            // not get a command to run, never a verdict on the patch — the
8299            // run is left exactly where a later reentry can retry again.
8300            E2eStatus::ResourceBlocked => {
8301                self.state.event(
8302                    "review",
8303                    format!(
8304                        "{why}; e2e could not run (resource unavailable); not \
8305                         deciding yet"
8306                    ),
8307                );
8308            }
8309            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8310                self.state.event(
8311                    "review",
8312                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8313                );
8314                self.record_contested_handoff();
8315                self.state.status = RunStatus::Gating;
8316            }
8317        }
8318        self.state.save()?;
8319        Ok(())
8320    }
8321
8322    // ----------------------------------------------------------------- gate
8323
8324    async fn gate(&mut self) -> Result<()> {
8325        if self.hold_base_sync().await? {
8326            return Ok(());
8327        }
8328        // Judged by the review record itself, not by `status`: a solo
8329        // candidate's `judge`/`deliberate` skip rewrites `status` on every
8330        // reentry (see `judge`), and trusting it here is exactly how a run
8331        // that exhausted its review budget got gated and merged a second
8332        // time around. `review_conclusion` recomputes the review loop's own
8333        // verdict from the round records themselves — `Gating` for a clean
8334        // round or a hand-off (see `stop_reviewing`), anything else means the
8335        // loop is still going or genuinely blocked.
8336        // A base the winner could not be replayed onto is a decision, not a
8337        // round: there is no landing tree to gate. Read as its own record for
8338        // the same reason the review verdict is.
8339        if self.state.status == RunStatus::Failed
8340            || self
8341                .state
8342                .base_sync
8343                .as_ref()
8344                .is_some_and(|s| s.conflict.is_some())
8345            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8346                != Some(RunStatus::Gating)
8347        {
8348            return Ok(());
8349        }
8350        let Some(winner) = self.state.winner().cloned() else {
8351            return Ok(());
8352        };
8353        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8354        self.state.reconcile_verification_head(&head);
8355        self.state.save()?;
8356        let wait = self.repository_wait();
8357        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8358            return Ok(());
8359        }
8360        if self.state.gate_ran {
8361            // `review_loop` derives its conclusion from the clean review
8362            // record on every reentry and therefore puts a completed run back
8363            // in `Gating`. A recorded gate is a stronger, terminal fact:
8364            // retain its original command output (or lack of any, for a repo
8365            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
8366            // doc) and restore `Blocked` on a real failure rather than
8367            // pretending the command is still running or running it a second
8368            // time. `gate_ran == false` remains the only shape — unattempted,
8369            // or a resource-blocked retry — that may still need to execute a
8370            // command.
8371            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8372                self.state.status = RunStatus::Blocked;
8373                self.state.save()?;
8374            }
8375            return Ok(());
8376        }
8377        self.state.status = RunStatus::Gating;
8378        let mut outcomes = self.run_gate(&winner).await?;
8379        let mut attempted_head = self.state.gate_head.clone();
8380        loop {
8381            // A resource-blocked outcome means the gate command never actually
8382            // ran - the shared build cache could not be acquired or confirmed
8383            // fresh in time - which is evidence about the machine, not about
8384            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
8385            // Recording it as a red gate would mark a run `Blocked` on nothing
8386            // but contention magi has already logged; leaving `self.state.gate`
8387            // empty and `self.state.gate_ran` false instead keeps the shape
8388            // this function already treats as "still needs to run" (see the
8389            // early-return above), so the next call retries the command
8390            // rather than concluding anything.
8391            if verify_inconclusive(&outcomes) {
8392                self.state.save()?;
8393                return Ok(());
8394            }
8395            if outcomes.iter().all(CommandOutcome::ok) {
8396                break;
8397            }
8398            let fixes_before = self.state.gate_fixes.len();
8399            match self.gate_fix_round(&winner, &outcomes).await? {
8400                GateFix::Retry => {
8401                    outcomes = self.run_gate(&winner).await?;
8402                    attempted_head = self.state.gate_head.clone();
8403                }
8404                GateFix::Stop => {
8405                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8406                    if self.state.gate_fixes.len() > fixes_before
8407                        && attempted_head.as_deref() != Some(current.as_str())
8408                        && verify_current_head(&mut self.state, &winner, &wait).await?
8409                    {
8410                        // A new SHA owes a gate even when a spent fixer changed only
8411                        // commit metadata. This does not grant another repair round.
8412                        outcomes = self.run_gate(&winner).await?;
8413                        attempted_head = self.state.gate_head.clone();
8414                    } else {
8415                        break;
8416                    }
8417                }
8418                GateFix::Defer => {
8419                    self.state.save()?;
8420                    return Ok(());
8421                }
8422            }
8423        }
8424        let passed = outcomes.iter().all(CommandOutcome::ok);
8425        self.state.gate = outcomes;
8426        self.state.gate_ran = true;
8427        self.state.gate_head = attempted_head;
8428        if !self.state.gate.is_empty()
8429            && !verify_inconclusive(&self.state.gate)
8430            && let Some(head) = self.state.gate_head.clone()
8431        {
8432            self.state.record_base_progress(&head);
8433        }
8434        self.state.save()?;
8435        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8436            return Ok(());
8437        }
8438        if !passed {
8439            self.state.status = RunStatus::Blocked;
8440            let spent = self.state.gate_fixes.len();
8441            self.state.event(
8442                "gate",
8443                if spent == 0 {
8444                    "gate failed; not merging".to_owned()
8445                } else {
8446                    format!("gate failed after {spent} gate-fix round(s); not merging")
8447                },
8448            );
8449        }
8450        self.state.save()?;
8451        Ok(())
8452    }
8453
8454    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
8455    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
8456    /// after review is clean and never on a candidate awaiting judging.
8457    ///
8458    /// Never fails the run: a non-zero exit or timeout is a warning and a
8459    /// recorded outcome, and the gate remains the single arbiter. Nothing
8460    /// configured means nothing happens - no event, no commit. `commit_all`
8461    /// commits any leftover change under the neutral identity and returns
8462    /// `false` when the tree is clean, so no empty commit is ever made.
8463    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8464        let commands = state.config.verify.pre_gate.clone();
8465        if commands.is_empty() {
8466            return;
8467        }
8468        let shell = state.config.shell();
8469        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8470        let (outcomes, _) = run_commands(
8471            state,
8472            "pre_gate",
8473            "pre_gate",
8474            0,
8475            &shell,
8476            &commands,
8477            &winner.worktree,
8478            timeout,
8479        )
8480        .await;
8481        for o in &outcomes {
8482            if !o.ok() {
8483                tracing::warn!(
8484                    "pre_gate `{}` failed ({:?}); the gate decides",
8485                    o.command,
8486                    o.code
8487                );
8488            }
8489            state.event(
8490                "pre_gate",
8491                format!(
8492                    "`{}` -> {}",
8493                    o.command,
8494                    if o.ok() {
8495                        "pass".to_owned()
8496                    } else {
8497                        format!(
8498                            "FAIL ({:?})\n{}",
8499                            o.code,
8500                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8501                        )
8502                    }
8503                ),
8504            );
8505        }
8506        state.pre_gate = outcomes;
8507        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8508            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8509                Ok(head) => {
8510                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8511                    state.pre_gate_commit = Some(head);
8512                }
8513                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8514            },
8515            Ok(false) => {}
8516            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8517        }
8518        if let Err(e) = state.save() {
8519            tracing::warn!("could not persist the pre_gate record: {e:#}");
8520        }
8521    }
8522
8523    /// Run `verify.gate` once against the winner's current tree, logging one
8524    /// event per command. Empty when nothing is configured.
8525    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8526        let wait = self.repository_wait();
8527        run_landing_gate(&mut self.state, winner, &wait).await
8528    }
8529
8530    /// One bounded fix round for a failing gate.
8531    ///
8532    /// The fixer is told the failure came from the gate itself, not from a
8533    /// reviewer, and is shown the failed commands, their exit codes and a tail
8534    /// of their output - whatever `[verify].gate` holds, nothing here knows
8535    /// what those commands run. Only a normal non-zero exit that printed
8536    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8537    /// command or a full disk says nothing about the code, and a fixer sent
8538    /// after it can only appease the machine. The round is judged by what git
8539    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8540    /// again before the gate does, so a fix cannot trade a green gate for a
8541    /// red e2e unnoticed.
8542    async fn gate_fix_round(
8543        &mut self,
8544        winner: &Candidate,
8545        outcomes: &[CommandOutcome],
8546    ) -> Result<GateFix> {
8547        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8548            g.phase == crate::repair_budget::Phase::Fixing
8549                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8550        }) {
8551            self.state.gate_fixes.len().saturating_add(1)
8552        } else {
8553            self.state.config.graph.gate_fix_rounds
8554        };
8555        let spent = self.state.gate_fixes.len();
8556        if spent >= cap {
8557            if cap > 0 {
8558                self.state.event(
8559                    "gate",
8560                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8561                );
8562            }
8563            return Ok(GateFix::Stop);
8564        }
8565        if !gate_fixable(outcomes) {
8566            self.state.event(
8567                "gate",
8568                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8569                 command or similar); not spending a fix round on it",
8570            );
8571            return Ok(GateFix::Stop);
8572        }
8573        let min_free = self.state.config.disk.min_free_bytes;
8574        if min_free > 0 {
8575            match crate::disk::free_bytes(&winner.worktree) {
8576                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8577                Ok(free) => {
8578                    self.state.event(
8579                        "gate",
8580                        format!(
8581                            "only {free} bytes free ({min_free} required by `[disk] \
8582                             min_free_bytes`); not spending a fix round on a failure the disk \
8583                             may explain"
8584                        ),
8585                    );
8586                    return Ok(GateFix::Stop);
8587                }
8588                Err(e) => {
8589                    self.state.event(
8590                        "gate",
8591                        format!("free disk space could not be measured ({e:#}); no fix round"),
8592                    );
8593                    return Ok(GateFix::Stop);
8594                }
8595            }
8596        }
8597
8598        let attempt = spent + 1;
8599        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8600        let base = self.landing_base();
8601        let fix_prompt = prompt::gate_fix_in(
8602            &self.state.instruction,
8603            &failed,
8604            attempt,
8605            cap,
8606            &self.state.config.graph.language,
8607            &self.state.config.graph.github_language,
8608        );
8609        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8610        let sessions = self.state.config.graph.sessions;
8611        let artifacts = self.merge_repair_artifacts();
8612        self.state.event(
8613            "gate",
8614            format!("gate failed; gate-fix round {attempt} of {cap}"),
8615        );
8616        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8617        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8618        // Persist before invoking the fixer: it may commit and then be
8619        // interrupted before its result (or the following e2e) is saved.
8620        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8621            head: before.clone(),
8622            passed: false,
8623            outcomes: Vec::new(),
8624        });
8625        self.state.save()?;
8626        let (_, seat, out) = self
8627            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8628                prompt: fix_prompt.clone(),
8629                spec,
8630                seat,
8631                cwd: winner.worktree.clone(),
8632                timeout,
8633                allow_write: true,
8634                sessions,
8635                artifacts: artifacts.clone(),
8636                stem: format!("gate-fix-{attempt}"),
8637                handover: None,
8638            })
8639            .await;
8640        // The fixer was never asked: no round is spent (nothing is recorded)
8641        // and a later reentry asks the same agent again.
8642        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8643            return Ok(GateFix::Defer);
8644        }
8645        let mut record = GateFixRecord {
8646            agent: seat.agent.clone(),
8647            failed,
8648            notes: String::new(),
8649            committed: false,
8650            error: None,
8651        };
8652        match out {
8653            AgentOutcome::Ok(o) => {
8654                // A missing report is not a failed fix: the round is judged
8655                // by the tree below, and the report only carries prose.
8656                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8657                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8658                }
8659            }
8660            AgentOutcome::Dropped(_) => {
8661                record.error = Some("the CLI dropped the stream".to_owned());
8662            }
8663            AgentOutcome::Quota(o) => {
8664                self.state.quota.push(QuotaLoss::from_report(
8665                    seat.key.clone(),
8666                    "gate-fix",
8667                    o.quota.as_ref(),
8668                ));
8669                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8670            }
8671            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8672        }
8673        self.state.seats.insert(seat.key.clone(), seat);
8674        let coordination = self.coordinate(&winner.worktree).await?;
8675        if let Ok(r) = git::rescue_commit(
8676            &winner.worktree,
8677            &format!("magi: gate fix {attempt} (uncommitted work)"),
8678        )
8679        .await
8680        {
8681            self.state.note_withheld("gate-fix", &r.withheld);
8682        }
8683        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8684        record.committed = after != before;
8685        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8686        let note = record.error.clone();
8687        self.state.gate_fixes.push(record);
8688        self.state.save()?;
8689        if !changed {
8690            self.state.event(
8691                "gate",
8692                match note {
8693                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8694                    None => format!("gate-fix round {attempt}: the tree did not change"),
8695                },
8696            );
8697            return Ok(GateFix::Stop);
8698        }
8699        self.state.event(
8700            "gate",
8701            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8702        );
8703
8704        drop(coordination);
8705        let wait = self.repository_wait();
8706        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8707    }
8708
8709    // ---------------------------------------------------------------- merge
8710
8711    /// One read-only rewrite by the summary's author; a text the gate still
8712    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8713    /// applies on their word or on silence. `None` means the run is parked on
8714    /// that question and nothing may be pushed or opened yet.
8715    async fn guarded_pr_message(
8716        &mut self,
8717        winner: &Candidate,
8718        facts: Option<&BranchFacts>,
8719        posting: bool,
8720    ) -> Result<Option<PrMessage>> {
8721        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8722        let head = match final_head.as_deref() {
8723            Some(h) => crate::pr_outcome::FinalHead::At(h),
8724            None => crate::pr_outcome::FinalHead::Unknown,
8725        };
8726        // Only a pull request that is about to be posted pays for a summary.
8727        let summarized = if posting {
8728            Some(self.summarize_change(winner, final_head.as_deref()).await)
8729        } else {
8730            None
8731        };
8732        // A busy shared resource is not a permanent cause: publishing an
8733        // "unavailable" description would stick (a later head move never
8734        // edits the body). Stop before anything is pushed or opened; the
8735        // run is `Blocked` and a resume asks the summary again.
8736        if self.state.resource_block.is_some() {
8737            self.state.event(
8738                "pr-summary",
8739                "a shared resource was busy; nothing was published, resume to summarize again",
8740            );
8741            return Ok(None);
8742        }
8743        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8744            anyhow::bail!(
8745                "the final commit moved during public description generation; retry before publishing"
8746            );
8747        }
8748        let note = match (&summarized, &self.state.change_summary) {
8749            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8750            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8751            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8752            (None, _) => ChangeNote::NotAsked,
8753        };
8754        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8755        if !posting {
8756            let identity = crate::scrub::Identity::current();
8757            return Ok(Some(PrMessage {
8758                title: crate::scrub::scrub(&pr.title, &identity),
8759                body: crate::scrub::scrub(&pr.body, &identity),
8760            }));
8761        }
8762        // This very text was already rejected and asked about: no second
8763        // judge call, rewrite or question, just carry the record on.
8764        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8765        if let Some(known) = self
8766            .state
8767            .github_text
8768            .clone()
8769            .filter(|g| g.fingerprint == raw_fp)
8770        {
8771            return self.settle_withheld(winner, &pr, known).await;
8772        }
8773        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8774        let mut violations =
8775            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8776        let identity = crate::scrub::Identity::current();
8777        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8778            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8779            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8780        {
8781            violations.push(crate::github_text::Violation::SensitiveData);
8782        }
8783        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8784        // only a language violation starts one; sensitive data alone goes
8785        // straight to the owner, with the redacted text as the fallback.
8786        let has_language = violations
8787            .iter()
8788            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8789        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8790            self.state.event(
8791                "github-text",
8792                if has_language {
8793                    format!("description rejected: {violations:?}; requesting one rewrite")
8794                } else {
8795                    format!("description rejected: {violations:?}; asking the owner")
8796                },
8797            );
8798            let mut rewritten = false;
8799            let mut rejected_rewrite = None;
8800            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8801            {
8802                let output = text;
8803                if let Some(title) = summary_title(&output) {
8804                    // Judge the redacted candidate that will actually be posted.
8805                    // Sensitive data alone must not discard a useful rewrite.
8806                    let raw_body = summary_without_title(&output);
8807                    let clean_title = crate::scrub::scrub(&title, &identity);
8808                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8809                    let redacted = clean_title != title || body != raw_body;
8810                    let title = clean_title;
8811                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8812                    if !complete {
8813                        self.state.event(
8814                            "github-text",
8815                            "description rewrite omitted required findings",
8816                        );
8817                    }
8818                    if !body.trim().is_empty() {
8819                        body.push_str("\n\n");
8820                        body.push_str(&public_pr_evidence(&self.state, head));
8821                    }
8822                    let marker = format!("magi:run/{}", self.state.id);
8823                    if !body.trim().is_empty() && !body.contains(&marker) {
8824                        body.push_str(&format!("\n\n{marker}"));
8825                    }
8826                    let rewrite = PrMessage {
8827                        title: title.clone(),
8828                        body: body.clone(),
8829                    };
8830                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8831                    let mut failed = crate::github_text::check_for(
8832                        &self.state.config.graph,
8833                        &title,
8834                        &body,
8835                        redecision,
8836                    );
8837                    if (!crate::github_text::shareable(&title)
8838                        || !crate::github_text::shareable(&body))
8839                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8840                    {
8841                        failed.push(crate::github_text::Violation::SensitiveData);
8842                    }
8843                    if complete
8844                        && !title.trim().is_empty()
8845                        && !body.trim().is_empty()
8846                        && !failed.is_empty()
8847                    {
8848                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8849                            title: title.clone(),
8850                            body: body.clone(),
8851                            failed: failed
8852                                .iter()
8853                                .map(|v| crate::github_text::category(*v).to_owned())
8854                                .collect(),
8855                        });
8856                    }
8857                    if complete
8858                        && !title.trim().is_empty()
8859                        && !raw_body.trim().is_empty()
8860                        && failed.is_empty()
8861                    {
8862                        decision = redecision;
8863                        pr = PrMessage { title, body };
8864                        rewritten = true;
8865                        self.state
8866                            .event("github-text", "description rewrite passed");
8867                        if redacted {
8868                            self.state.event(
8869                                "github-text",
8870                                "sensitive data removed from description rewrite",
8871                            );
8872                        }
8873                    }
8874                }
8875            }
8876            if !rewritten {
8877                self.state.event(
8878                    "github-text",
8879                    "description rewrite unavailable or rejected; asking the owner",
8880                );
8881                if let Some(w) =
8882                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8883                {
8884                    // A different text than the one asked about before: that
8885                    // question no longer describes anything.
8886                    self.retire_github_text_question("the withheld text changed");
8887                    let record = crate::run::GithubTextAsk {
8888                        rewrite: rejected_rewrite,
8889                        fingerprint: raw_fp,
8890                        title: w.title,
8891                        body: w.body,
8892                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8893                        question: None,
8894                        asks: 0,
8895                        resolved: false,
8896                        chosen_title: None,
8897                        choice: None,
8898                        publication: None,
8899                    };
8900                    return self.settle_withheld(winner, &pr, record).await;
8901                }
8902            }
8903        }
8904        let (title, body) =
8905            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8906        Ok(Some(PrMessage { title, body }))
8907    }
8908
8909    /// Describe the whole change from its final diff in either run mode.
8910    ///
8911    /// The summary is tied to the final commit and to the merge base of the
8912    /// resolved remote base; a record for any other pair is dropped and the
8913    /// agent asked again, so a description never outlives the diff it was
8914    /// written from. The seat is read-only and the prompt tells it to read the
8915    /// diff and the code around it. `[roles] synthesizer` selects the agents
8916    /// (unset: the default pick), each asked at most once, advancing on an
8917    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8918    /// carries the reason it is unavailable; nothing is invented then.
8919    async fn summarize_change(
8920        &mut self,
8921        winner: &Candidate,
8922        head: Option<&str>,
8923    ) -> std::result::Result<(), String> {
8924        let Some(head) = head else {
8925            return Err("the final commit could not be read".to_owned());
8926        };
8927        let remote = self.state.config.merge.remote.clone();
8928        let tracking = format!("{remote}/{}", self.state.base_branch);
8929        if !git::rev_exists(&winner.worktree, &tracking).await {
8930            return Err(format!("the remote base `{tracking}` could not be read"));
8931        }
8932        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8933            Ok(b) if !b.is_empty() => b,
8934            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8935        };
8936        let evidence = crate::pr_outcome::summary_evidence(
8937            &self.state,
8938            crate::pr_outcome::FinalHead::At(head),
8939        );
8940        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8941        if self
8942            .state
8943            .change_summary
8944            .as_ref()
8945            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8946        {
8947            return Ok(());
8948        }
8949        if self.state.change_summary.take().is_some() {
8950            self.state.event(
8951                "pr-summary",
8952                "the final diff or evidence changed; the earlier change summary was discarded",
8953            );
8954        }
8955        // Save the source snapshot even if no generation agent is available.
8956        if let Err(e) = write_artifact(
8957            &self.state,
8958            &format!("public-description-{fingerprint}.md"),
8959            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8960        ) {
8961            tracing::warn!("could not save public description evidence: {e:#}");
8962        }
8963        let chain = match agent::pick_chain(
8964            &self.state.config.agents,
8965            self.state.config.roles.synthesizer.as_ref(),
8966            &agent::installed,
8967            "synthesizer",
8968        ) {
8969            Ok(chain) => chain,
8970            Err(e) => {
8971                tracing::warn!("no agent for the change summary: {e:#}");
8972                self.state.event("pr-summary", "no summary agent available");
8973                return Err("no agent was available to summarize it".to_owned());
8974            }
8975        };
8976        let subjects = git::subjects(&winner.worktree, &base, head)
8977            .await
8978            .unwrap_or_default();
8979        let stat = git::diff_stat(&winner.worktree, &base, head)
8980            .await
8981            .unwrap_or_default();
8982        let stat: String = stat.chars().take(6_000).collect();
8983        let mut prompt = crate::prompt::change_summary_in(
8984            &base,
8985            head,
8986            &subjects,
8987            stat.trim(),
8988            &self.state.config.graph.github_language,
8989        );
8990        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8991        let sessions = self.state.config.graph.sessions;
8992        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8993        let prompts = self.state.config.prompts.clone();
8994        let cache = self.state.config.cache_dir();
8995        let run = self.state.id.clone();
8996        for (i, spec) in chain.into_iter().enumerate() {
8997            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8998            let github_spec = match agent::for_github_text(
8999                &spec,
9000                &self.state.config.graph.github_language,
9001                &winner.worktree,
9002            ) {
9003                Ok(spec) => spec,
9004                Err(e) => {
9005                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9006                    continue;
9007                }
9008            };
9009            let job = SeatJob {
9010                spec: github_spec,
9011                seat,
9012                cwd: winner.worktree.clone(),
9013                prompt: prompt.clone(),
9014                timeout,
9015                allow_write: false,
9016                sessions,
9017                artifacts: agent::artifacts_dir(&self.state.dir()),
9018                stem: if i == 0 {
9019                    "change-summary".to_owned()
9020                } else {
9021                    format!("change-summary-{}", spec.id)
9022                },
9023                handover: None,
9024            };
9025            let ctx = WaveCtx {
9026                run: &run,
9027                node: "pr-summary",
9028                prompts: &prompts,
9029                cache: cache.as_deref(),
9030                round: None,
9031                carry_seats: false,
9032                pauses: Some((&self.pause, &self.interrupt)),
9033            };
9034            let (seat, outcome) =
9035                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9036            self.state.seats.insert(seat.key.clone(), seat);
9037            let text = match outcome {
9038                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9039                AgentOutcome::ResourceBlocked(why) => {
9040                    // No agent was asked: the run stops resumably before
9041                    // anything is published (see `guarded_pr_message`).
9042                    if self.state.resource_block.is_none() {
9043                        self.state.resource_block = Some(crate::run::ResourceBlock {
9044                            cause: crate::run::ResourceBlockCause::Other,
9045                            node: "pr-summary".to_owned(),
9046                            kind: "shared resource".to_owned(),
9047                            detail: why,
9048                            at: Timestamp::now(),
9049                        });
9050                    }
9051                    return Err("a shared resource was busy".to_owned());
9052                }
9053                _ => continue,
9054            };
9055            let blind = &self.state.config.blind;
9056            let Some(title) = summary_title(&text) else {
9057                continue;
9058            };
9059            let body: String = summary_without_title(&text)
9060                .lines()
9061                .filter(|l| !l.contains("magi:run/"))
9062                .collect::<Vec<_>>()
9063                .join("\n");
9064            let body = blind::sanitize_prose(body.trim(), blind);
9065            let title = blind::sanitize_prose(&title, blind);
9066            if body.trim().is_empty() || title.trim().is_empty() {
9067                continue;
9068            }
9069            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9070                continue;
9071            }
9072            // The branch must still be the one the agent was shown.
9073            if git::rev_parse(&winner.worktree, "HEAD")
9074                .await
9075                .ok()
9076                .as_deref()
9077                != Some(head)
9078            {
9079                return Err("the branch moved while the summary was written".to_owned());
9080            }
9081            if git::merge_base(&winner.worktree, &tracking, head)
9082                .await
9083                .ok()
9084                .as_deref()
9085                != Some(base.as_str())
9086                || crate::pr_outcome::summary_evidence(
9087                    &self.state,
9088                    crate::pr_outcome::FinalHead::At(head),
9089                ) != evidence
9090            {
9091                return Err("the summary evidence changed while it was written".to_owned());
9092            }
9093            self.state.change_summary = Some(crate::run::ChangeSummary {
9094                head: head.to_owned(),
9095                base,
9096                evidence: fingerprint,
9097                title,
9098                body,
9099            });
9100            self.state.event(
9101                "pr-summary",
9102                format!("change summary written by {} for the final diff", spec.id),
9103            );
9104            if let Err(e) = self.state.save() {
9105                tracing::warn!("could not save the change summary: {e:#}");
9106            }
9107            return Ok(());
9108        }
9109        self.state
9110            .event("pr-summary", "no agent produced a change summary");
9111        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9112    }
9113
9114    /// One read-only rewrite request for a withheld description. The
9115    /// implementer's own conversation is tried first when it exists and can be
9116    /// resumed; then the `[roles] language_judge` chain (or the default pick)
9117    /// writes it in a separate conversation, skipping the implementer's agent.
9118    /// The prompt carries the whole text, so no agent needs any memory. Each
9119    /// agent is asked at most once and the chain advances on an error, quota,
9120    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
9121    /// that has run out of quota must not end the rewrite. Nothing is recorded
9122    /// as a handover.
9123    async fn rewrite_pr_text(
9124        &mut self,
9125        winner: &Candidate,
9126        pr: &PrMessage,
9127        violations: &[crate::github_text::Violation],
9128    ) -> Option<String> {
9129        let prompt = format!(
9130            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9131            pr.title, pr.body
9132        );
9133        let sessions = self.state.config.graph.sessions;
9134        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9135        // (spec, the resumed implementer's seat). A fallback seat is created
9136        // only when its turn comes, so an unused fallback leaves no record.
9137        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9138        let key = format!("impl-{}", winner.label);
9139        if let Some(spec) = self
9140            .state
9141            .config
9142            .agents
9143            .iter()
9144            .find(|a| a.id == winner.agent)
9145            .cloned()
9146            && let Some(existing) = self.state.seats.get(&key)
9147            && existing.agent == spec.id
9148            && agent::has_session(spec.kind, existing, sessions)
9149        {
9150            attempts.push((spec, Some(existing.clone())));
9151        }
9152        match agent::pick_chain(
9153            &self.state.config.agents,
9154            self.state.config.roles.language_judge.as_ref(),
9155            &agent::installed,
9156            "language_judge",
9157        ) {
9158            Ok(chain) => {
9159                for spec in chain {
9160                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9161                        continue;
9162                    }
9163                    attempts.push((spec, None));
9164                }
9165            }
9166            Err(e) if attempts.is_empty() => {
9167                tracing::warn!("no agent for the description rewrite: {e:#}");
9168                self.state
9169                    .event("github-text", "no rewrite agent available");
9170                return None;
9171            }
9172            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9173        }
9174        let prompts = self.state.config.prompts.clone();
9175        let cache = self.state.config.cache_dir();
9176        let run = self.state.id.clone();
9177        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9178            let resumed = resumed_seat.is_some();
9179            let seat = resumed_seat.unwrap_or_else(|| {
9180                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9181            });
9182            let github_spec = match agent::for_github_text(
9183                &spec,
9184                &self.state.config.graph.github_language,
9185                &winner.worktree,
9186            ) {
9187                Ok(spec) => spec,
9188                Err(e) => {
9189                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9190                    continue;
9191                }
9192            };
9193            let job = SeatJob {
9194                spec: github_spec,
9195                seat,
9196                cwd: winner.worktree.clone(),
9197                prompt: prompt.clone(),
9198                timeout: retry_budget(timeout_full, resumed),
9199                allow_write: false,
9200                sessions,
9201                artifacts: agent::artifacts_dir(&self.state.dir()),
9202                stem: if i == 0 {
9203                    "github-text-rewrite".to_owned()
9204                } else {
9205                    format!("github-text-rewrite-{}", spec.id)
9206                },
9207                handover: None,
9208            };
9209            let ctx = WaveCtx {
9210                run: &run,
9211                node: "github-text",
9212                prompts: &prompts,
9213                cache: cache.as_deref(),
9214                round: None,
9215                carry_seats: false,
9216                pauses: Some((&self.pause, &self.interrupt)),
9217            };
9218            let (seat, outcome) =
9219                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9220            self.state.seats.insert(seat.key.clone(), seat);
9221            match outcome {
9222                AgentOutcome::Ok(o)
9223                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9224                {
9225                    return Some(o.text);
9226                }
9227                AgentOutcome::ResourceBlocked(_) => return None,
9228                _ => {}
9229            }
9230        }
9231        None
9232    }
9233
9234    /// Abandon the standing posting-gate question, if any.
9235    fn retire_github_text_question(&mut self, why: &str) {
9236        let Some(id) = self
9237            .state
9238            .github_text
9239            .as_ref()
9240            .and_then(|g| g.question.clone())
9241        else {
9242            return;
9243        };
9244        let _ = ask::Questions::open().update(&id, |q| {
9245            q.abandon(why);
9246            Ok(())
9247        });
9248    }
9249
9250    /// Carry a withheld text to its end: file the question, park while it is
9251    /// open, and on an answer, silence or timeout pick the text to post. The
9252    /// decision is saved before it is used, so it applies once.
9253    async fn settle_withheld(
9254        &mut self,
9255        winner: &Candidate,
9256        pr: &PrMessage,
9257        mut g: crate::run::GithubTextAsk,
9258    ) -> Result<Option<PrMessage>> {
9259        use crate::github_text as gt;
9260        let store = ask::Questions::open();
9261        let timeout = self.state.config.graph.answer_timeout;
9262        if !g.resolved {
9263            let mut standing = g
9264                .question
9265                .as_ref()
9266                .and_then(|id| store.list_for_effects().into_iter().find(|q| &q.id == id));
9267            if let Some(q) = standing.as_ref()
9268                && q.status.open()
9269                && !store.held_reply(&q.id)
9270                && gt::expired(q, timeout)
9271            {
9272                let why = format!("no answer within {}s of asking", timeout.max(1));
9273                standing = store
9274                    .update(&q.id, |q| {
9275                        q.abandon(&why);
9276                        Ok(())
9277                    })
9278                    .ok()
9279                    .map(|(q, ())| q);
9280            }
9281            let mut ask_again: Option<Option<Vec<String>>> = None;
9282            match standing {
9283                // Recorded but never written (a stop between the two saves),
9284                // or removed: ask again without spending the retry.
9285                None if g.asks <= 1 => {
9286                    g.asks = 0;
9287                    ask_again = Some(None);
9288                }
9289                None => {
9290                    g.resolved = true;
9291                    g.chosen_title = None;
9292                    g.choice = Some(GithubTextChoice::Fallback);
9293                }
9294                Some(q) if q.status.open() => {
9295                    self.park_on_github_text(&g)?;
9296                    return Ok(None);
9297                }
9298                Some(q) => match gt::read_reply(&q) {
9299                    gt::Reply::Fallback => {
9300                        g.resolved = true;
9301                        g.chosen_title = None;
9302                        g.choice = Some(GithubTextChoice::Fallback);
9303                    }
9304                    gt::Reply::Original => {
9305                        g.resolved = true;
9306                        g.chosen_title = None;
9307                        g.choice = Some(GithubTextChoice::Original);
9308                    }
9309                    gt::Reply::Rewrite => {
9310                        g.resolved = true;
9311                        g.chosen_title = None;
9312                        g.choice = Some(if g.rewrite.is_some() {
9313                            GithubTextChoice::Rewrite
9314                        } else {
9315                            GithubTextChoice::Fallback
9316                        });
9317                    }
9318                    gt::Reply::Title(text) => {
9319                        let first = text
9320                            .lines()
9321                            .map(str::trim)
9322                            .find(|l| !l.is_empty())
9323                            .unwrap_or("");
9324                        let decision = if first.is_empty() {
9325                            None
9326                        } else {
9327                            let candidate = PrMessage {
9328                                title: first.to_owned(),
9329                                body: String::new(),
9330                            };
9331                            self.judge_pr_language(&winner.worktree, &candidate).await
9332                        };
9333                        match gt::vet_title(
9334                            &text,
9335                            decision,
9336                            self.state.config.graph.github_english(),
9337                        ) {
9338                            Ok(title) => {
9339                                g.resolved = true;
9340                                g.chosen_title = Some(title);
9341                                g.choice = Some(GithubTextChoice::MyTitle);
9342                            }
9343                            Err(cats) if g.asks < 2 => {
9344                                ask_again = Some(Some(
9345                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9346                                ))
9347                            }
9348                            Err(_) => {
9349                                g.resolved = true;
9350                                g.chosen_title = None;
9351                                g.choice = Some(GithubTextChoice::Fallback);
9352                            }
9353                        }
9354                    }
9355                },
9356            }
9357            if let Some(rejected) = ask_again {
9358                // A retry (after a rejected replacement title) is still about
9359                // the original text: the same ORIGINAL / FALLBACK comparison.
9360                let w = gt::Withheld {
9361                    title: g.title,
9362                    body: g.body,
9363                    categories: g.categories.clone(),
9364                };
9365                // Named by the original text's fingerprint alone, so a resume
9366                // rewrites the same file and the detail stays identical.
9367                let artifact = match crate::run::write_artifact(
9368                    &self.state,
9369                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
9370                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9371                ) {
9372                    Ok(p) => Some(p),
9373                    Err(e) => {
9374                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
9375                        None
9376                    }
9377                };
9378                let (detail, comparison) = gt::question_content(
9379                    &w,
9380                    &pr.title,
9381                    &pr.body,
9382                    &self.state.id,
9383                    rejected.as_deref(),
9384                    artifact.as_deref(),
9385                    g.rewrite.as_ref(),
9386                );
9387                let mut q = ask::Question::new(
9388                    self.state.id.clone(),
9389                    gt::ASK_NODE.to_owned(),
9390                    gt::ASK_SEAT.to_owned(),
9391                    gt::question_summary(&self.state.config.graph.language, &w),
9392                    detail,
9393                    gt::question_choices(&w, g.rewrite.as_ref()),
9394                );
9395                q.posting_comparison = Some(comparison);
9396                q.answer_timeout = timeout;
9397                // Recorded before it is written: a stop in between leaves a
9398                // record naming a question the resume then files again.
9399                g.question = Some(q.id.clone());
9400                g.asks += 1;
9401                self.state.github_text = Some(g.clone());
9402                self.state.event(
9403                    "github-text",
9404                    format!("asking the owner about the withheld text ({})", q.short()),
9405                );
9406                self.state.save()?;
9407                store
9408                    .put_executable(&mut q)
9409                    .context("file the posting-gate question")?;
9410                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9411                    tracing::warn!(
9412                        "could not notify about posting-gate question {}: {e:#}",
9413                        q.short()
9414                    );
9415                }
9416                self.park_on_github_text(&g)?;
9417                return Ok(None);
9418            }
9419        }
9420        let resolved = g.chosen_title.clone();
9421        let original = g.choice == Some(GithubTextChoice::Original);
9422        // Field names only: the event log never records the text.
9423        self.state.event(
9424            "github-text",
9425            if original {
9426                format!(
9427                    "posting the original text for: {}",
9428                    match (g.title, g.body) {
9429                        (true, true) => "title, description",
9430                        (true, false) => "title",
9431                        _ => "description",
9432                    }
9433                )
9434            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9435                "posting the saved rewrite title and description".to_owned()
9436            } else if resolved.is_some() {
9437                "posting the owner's replacement title".to_owned()
9438            } else {
9439                "posting the neutral text for the withheld fields".to_owned()
9440            },
9441        );
9442        self.state.github_text = Some(g.clone());
9443        self.state.save()?;
9444        if g.choice == Some(GithubTextChoice::Rewrite)
9445            && let Some(r) = g.rewrite.as_ref()
9446        {
9447            return Ok(Some(PrMessage {
9448                title: r.title.clone(),
9449                body: r.body.clone(),
9450            }));
9451        }
9452        let w = gt::Withheld {
9453            title: g.title,
9454            body: g.body,
9455            categories: g.categories.clone(),
9456        };
9457        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9458        if let Some(t) = resolved {
9459            title = t;
9460        }
9461        // Every withheld field is now neutral or vetted English.
9462        let english = gt::LanguageDecision {
9463            title_english: true,
9464            body_english: true,
9465            body_complete: true,
9466        };
9467        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9468        if original {
9469            // The owner's explicit choice overrides the gate for the fields it
9470            // was made for; the others keep what the gate produced.
9471            if g.title {
9472                title = pr.title.clone();
9473            }
9474            if g.body {
9475                body = pr.body.clone();
9476            }
9477        }
9478        Ok(Some(PrMessage { title, body }))
9479    }
9480
9481    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9482        self.state.github_text = Some(g.clone());
9483        self.state.end_pass_parked(ParkCause::Question);
9484        self.state.parked = true;
9485        self.state.event(
9486            "github-text",
9487            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9488        );
9489        self.state.save()
9490    }
9491
9492    /// Ask `[roles] language_judge` about this text and record which source
9493    /// decided; the text itself is never recorded.
9494    async fn judge_pr_language(
9495        &mut self,
9496        cwd: &Path,
9497        pr: &PrMessage,
9498    ) -> Option<crate::github_text::LanguageDecision> {
9499        self.state.config.roles.language_judge.as_ref()?;
9500        if !self.state.config.graph.language_gate() {
9501            return None;
9502        }
9503        let decision =
9504            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9505        self.state.event(
9506            "github-text",
9507            if decision.is_some() {
9508                "language decided by the language judge"
9509            } else {
9510                "language judge unavailable; using built-in heuristics"
9511            },
9512        );
9513        decision
9514    }
9515
9516    fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9517        Box::pin(self.merge_inner())
9518    }
9519
9520    async fn merge_inner(&mut self) -> Result<()> {
9521        if self.resource_halt()? || self.hold_base_sync().await? {
9522            return Ok(());
9523        }
9524        // Same reasoning as `gate`: ask the review and gate records directly
9525        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9526        // skip can rewrite on reentry to something that no longer says
9527        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9528        // so a hand-off (open findings, green verification) reaches merge
9529        // exactly like a genuinely clean round does.
9530        //
9531        // A run resumed mid-`land` never reaches here at all: `execute`
9532        // recognises `RunStatus::Landing` before it even calls `prep`, and
9533        // routes straight to `run_land` instead. That has to happen a level
9534        // up from this function, not with a check in here, because
9535        // `review_loop`'s own status recomputation (see its doc) runs
9536        // *before* `merge` on every reentry and would otherwise overwrite
9537        // the `Landing` marker with `Gating` before this node ever saw it.
9538        if self
9539            .state
9540            .base_sync
9541            .as_ref()
9542            .is_some_and(|s| s.conflict.is_some())
9543            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9544                != Some(RunStatus::Gating)
9545            // `gate_ran == false` is not "passed" - `gate` leaves it false
9546            // both before it has ever run and when its last attempt was
9547            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9548            // permission to merge on nothing but the review record. Only a
9549            // gate that actually ran may proceed; an intentional local
9550            // review can also finish with verification explicitly not configured.
9551            // `RunState::gate_status` computes that distinction.
9552            || !self.state.gate_allows_progress()
9553        {
9554            return Ok(());
9555        }
9556        // This node's own record, not `status`: `status == Ready` is not
9557        // unique to the harmless `MergeMode::None` path this line was
9558        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9559        // run's PR was closed without merging — and on that run `mode` is
9560        // still `Pr`, so a reentry that fell through here would push and
9561        // open a second pull request. `self.state.merge` is set exactly once
9562        // this node (or `land`) has already produced a verdict, under every
9563        // mode, which is what "already done" actually means here.
9564        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9565            return Ok(());
9566        }
9567        let Some(winner) = self.state.winner().cloned() else {
9568            return Ok(());
9569        };
9570        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9571        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9572            self.state.status = RunStatus::Blocked;
9573            self.state.event(
9574                "gate",
9575                "winner changed after gate; fresh verification required",
9576            );
9577            self.state.save()?;
9578            return Ok(());
9579        }
9580        let wait = self.repository_wait();
9581        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9582            return Ok(());
9583        }
9584        let repo = self.state.repo.clone();
9585        let base = self.state.base_branch.clone();
9586        let mode = self.state.config.merge.mode;
9587        let style = self.state.config.merge.style;
9588        let facts = if is_review_run(&self.state) {
9589            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9590            let start = review_base(
9591                &repo,
9592                &self.state.config.merge.remote,
9593                &base,
9594                &self.state.base_commit,
9595                &winner.branch,
9596            )
9597            .await;
9598            branch_facts(&repo, &start, &winner.branch).await
9599        } else {
9600            None
9601        };
9602        // `None` when the base could not be freshly read: then an adopted
9603        // pull request's title is left alone.
9604        let leaked = if is_review_run(&self.state) {
9605            leaked_subjects(&self.state, &winner.branch).await
9606        } else {
9607            Some(Vec::new())
9608        };
9609        // `None`: parked on the owner's word about a withheld title or body.
9610        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9611        // the resume arrives here again.
9612        let Some(pr) = self
9613            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9614            .await?
9615        else {
9616            return Ok(());
9617        };
9618        let message = pr.commit_message();
9619
9620        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9621        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9622            return Ok(());
9623        }
9624        let outcome = match mode {
9625            MergeMode::None => MergeOutcome {
9626                mode,
9627                ok: true,
9628                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9629                empty: false,
9630            },
9631            MergeMode::Pr | MergeMode::Local
9632                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9633            {
9634                MergeOutcome {
9635                    mode,
9636                    ok: false,
9637                    detail: empty_candidate_detail(&self.state, &base),
9638                    empty: true,
9639                }
9640            }
9641            MergeMode::Local => {
9642                let on = git::current_branch(&repo).await?;
9643                if on.as_deref() != Some(base.as_str()) {
9644                    MergeOutcome {
9645                        mode,
9646                        ok: false,
9647                        detail: format!(
9648                            "{} has {} checked out, not the base branch {base}",
9649                            repo.display(),
9650                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9651                        ),
9652                        empty: false,
9653                    }
9654                } else if !git::is_clean(&repo).await? {
9655                    MergeOutcome {
9656                        mode,
9657                        ok: false,
9658                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9659                        empty: false,
9660                    }
9661                } else {
9662                    let out = match style {
9663                        MergeStyle::Merge => {
9664                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9665                        }
9666                        MergeStyle::Squash => {
9667                            git::merge_squash(&repo, &winner.branch, &message).await?
9668                        }
9669                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9670                    };
9671                    MergeOutcome {
9672                        mode,
9673                        ok: out.ok(),
9674                        detail: if out.ok() { out.stdout } else { out.stderr },
9675                        empty: false,
9676                    }
9677                }
9678            }
9679            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9680                self.state.pr_publication_pending = true;
9681                self.state.save()?;
9682                let out =
9683                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9684                self.state.pr_publication_pending = out.is_err();
9685                MergeOutcome {
9686                    mode,
9687                    ok: out.is_ok(),
9688                    detail: match out {
9689                        Ok(url) => url,
9690                        Err(e) => format!("{e:#}"),
9691                    },
9692                    empty: false,
9693                }
9694            }
9695            MergeMode::Pr => {
9696                let remote = self.state.config.merge.remote.clone();
9697                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9698                let pushed = if self.state.foreign_checkout.is_some() {
9699                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9700                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9701                } else {
9702                    git::push(&winner.worktree, &remote, &winner.branch).await?
9703                };
9704                if !pushed.ok() {
9705                    MergeOutcome {
9706                        mode,
9707                        ok: false,
9708                        detail: pushed.stderr,
9709                        empty: false,
9710                    }
9711                } else {
9712                    if self.state.foreign_checkout.is_some() {
9713                        self.state.pr_pushed_head = Some(delivery_head.clone());
9714                    }
9715                    // A retry or resume of a run whose branch already has an
9716                    // open pull request adopts it rather than failing on a
9717                    // duplicate. Only this winner branch into this base:
9718                    // `branch_for` derives the name from the run id, so a
9719                    // different run's pull request never matches.
9720                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9721                    let out = match pr_merge_plan(found) {
9722                        PrPlan::Create => {
9723                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9724                                .await
9725                        }
9726                        PrPlan::Adopt { url, title } => {
9727                            self.state
9728                                .event("merge", format!("Pr: adopted open pull request {url}"));
9729                            if self.state.explicit_github_text_selection() {
9730                                // An explicit publication selection applies the prepared
9731                                // payload; acceptance alone is not publication proof.
9732                                land::set_pr_text_verbatim(
9733                                    &winner.worktree,
9734                                    &url,
9735                                    &pr.title,
9736                                    &pr.body,
9737                                )
9738                                .await
9739                                .map(|()| url)
9740                            } else {
9741                                if title != pr.title
9742                                    && (!is_review_run(&self.state)
9743                                        || leaked
9744                                            .as_deref()
9745                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9746                                    && let Err(e) =
9747                                        if self.state.github_text.as_ref().is_some_and(|g| {
9748                                            g.resolved
9749                                                && g.title
9750                                                && g.choice == Some(GithubTextChoice::Original)
9751                                        }) {
9752                                            // The owner chose to post this title unmodified.
9753                                            land::set_pr_title_verbatim(
9754                                                &winner.worktree,
9755                                                &url,
9756                                                &pr.title,
9757                                            )
9758                                            .await
9759                                        } else {
9760                                            land::set_pr_title(
9761                                                &mut self.state,
9762                                                &winner.worktree,
9763                                                &url,
9764                                                &pr.title,
9765                                            )
9766                                            .await
9767                                        }
9768                                {
9769                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9770                                    self.state
9771                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9772                                }
9773                                Ok(url)
9774                            }
9775                        }
9776                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9777                    };
9778                    match out {
9779                        Ok(url) => MergeOutcome {
9780                            mode,
9781                            ok: true,
9782                            detail: url,
9783                            empty: false,
9784                        },
9785                        Err(e) => MergeOutcome {
9786                            mode,
9787                            ok: false,
9788                            detail: e.to_string(),
9789                            empty: false,
9790                        },
9791                    }
9792                }
9793            }
9794        };
9795
9796        self.state.status = match (mode, outcome.ok) {
9797            (MergeMode::None, _) => RunStatus::Ready,
9798            (_, true) if outcome.empty => RunStatus::Ready,
9799            (MergeMode::Pr, true) => RunStatus::Ready,
9800            (_, true) => RunStatus::Merged,
9801            (_, false) => RunStatus::Blocked,
9802        };
9803        self.state.event(
9804            "merge",
9805            format!(
9806                "{:?}: {}",
9807                mode,
9808                outcome.detail.lines().next().unwrap_or("")
9809            ),
9810        );
9811        if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9812            self.state.record_github_text_publication(&outcome.detail);
9813        }
9814        self.state.merge = Some(outcome);
9815        self.state.save()?;
9816
9817        // The PR is open and the run would historically stop here, leaving the
9818        // operator to watch checks, feed review comments back to a fixer, and
9819        // merge. That was done by hand six times in one session before this
9820        // existed. Opt-in, because merging is the one irreversible thing magi
9821        // can do to a repository.
9822        if self.state.config.graph.land
9823            && mode == MergeMode::Pr
9824            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9825        {
9826            self.run_land().await?;
9827        }
9828        // `run_land` may have left `status` at `Landing` - still waiting on
9829        // CI or the owner's approval, not actually settled - so this has to
9830        // read whatever `status` ended up as here, not the `Merged` this
9831        // function set a few lines up.
9832        self.settle_questions();
9833        Ok(())
9834    }
9835
9836    /// Enter `land`.
9837    ///
9838    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9839    /// a resumed run's re-entry. `land::land` itself is what serialises the
9840    /// two git-mutating moments inside the loop — the rebase push and
9841    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9842    /// to hold a lock across the whole call, and doing so would serialise
9843    /// this run's CI wait against a *different* run's land-approval resume
9844    /// in the same repository, which is exactly the "must not wait on
9845    /// another task" property the daemon's slot-freeing exists to give.
9846    fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9847        Box::pin(self.run_land_inner())
9848    }
9849
9850    async fn run_land_inner(&mut self) -> Result<()> {
9851        crate::pr_head::ensure_owner(&self.state)?;
9852        let url = self
9853            .state
9854            .pr
9855            .as_ref()
9856            .map(|pr| pr.url.clone())
9857            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9858            .unwrap_or_default();
9859        let url = url.lines().next().unwrap_or("").trim().to_owned();
9860        if !url.starts_with("http") {
9861            return Ok(());
9862        }
9863        // A land failure is not a lost run: the work is on a branch and the
9864        // pull request is open, which is exactly where a human takes over.
9865        loop {
9866            match land::land(&mut self.state, &url).await {
9867                Ok(_)
9868                    if self
9869                        .state
9870                        .merge_repairs
9871                        .last()
9872                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9873                {
9874                    if self.repair_merge_review().await? {
9875                        continue;
9876                    }
9877                }
9878                Ok(_) if self.state.resource_block.is_some() => {
9879                    return self.stop_for_resource();
9880                }
9881                Ok(pr) if self.state.parked => {
9882                    // `land` already saved the parked marker; nothing here
9883                    // overrides `status` back to a terminal value while an
9884                    // approval is still outstanding.
9885                    let _ = pr;
9886                }
9887                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9888                    self.state.status = RunStatus::Superseded;
9889                    self.state.save()?;
9890                }
9891                Ok(pr) => {
9892                    self.state.status = match pr.state {
9893                        land::PrLifecycle::Merged => RunStatus::Merged,
9894                        _ => RunStatus::Blocked,
9895                    };
9896                    // Downstream of a confirmed merge only - see
9897                    // `bump::should_release_bump`'s own doc for why this one
9898                    // check covers all three of `land`'s success paths.
9899                    // Best-effort: the run already landed, so a failure here
9900                    // (the decision call, `gh`, `cargo`) is recorded and never
9901                    // turns a landed run into a failed one.
9902                    if bump::should_release_bump(self.state.status)
9903                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9904                    {
9905                        // The event is the run's own record. Not-eligible cases
9906                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9907                        // `Err` is a bump that was tried and failed:
9908                        // `after_merge` itself raises the operator notice for
9909                        // that, whether or not a release PR exists yet.
9910                        self.state
9911                            .event("bump", format!("release bump skipped: {e:#}"));
9912                    }
9913                    // Independent of the bump, and best-effort in the same way:
9914                    // findings the merge left open become follow-up tasks.
9915                    if self.state.status == RunStatus::Merged {
9916                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9917                    }
9918                    self.state.save()?;
9919                }
9920                Err(e) => {
9921                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9922                        return Err(e);
9923                    }
9924                    self.state.status = RunStatus::Blocked;
9925                    self.state.event("land", format!("gave up: {e}"));
9926                    self.state.save()?;
9927                }
9928            }
9929            break;
9930        }
9931        Ok(())
9932    }
9933
9934    /// One repair cycle stays under the existing driver/task claim. An
9935    /// interrupted fixer is rolled back and held rather than run twice.
9936    async fn repair_merge_review(&mut self) -> Result<bool> {
9937        use crate::run::MergeRepairPhase as Phase;
9938        crate::pr_head::ensure_owner(&self.state)?;
9939        let i = self.state.merge_repairs.len() - 1;
9940        let request = self.state.merge_repairs[i].clone();
9941        if request.phase == Phase::Fixing {
9942            let restored = match self.state.winner().cloned() {
9943                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9944                None => Err(anyhow::anyhow!("repair has no winning branch")),
9945            };
9946            let why = match restored {
9947                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9948                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9949                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9950            };
9951            self.hold_merge_repair(i, why)?;
9952            return Ok(false);
9953        }
9954        // A completed repair gate is a publication checkpoint. On reentry,
9955        // reject external changes before the ordinary gate path can replace
9956        // that evidence by verifying a new head. Requested repairs still
9957        // start their own review generation below.
9958        if request.phase == Phase::Reviewing && self.state.gate_ran {
9959            let winner = self
9960                .state
9961                .winner()
9962                .context("repair has no winning branch")?;
9963            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9964            if self.state.gate_head.as_deref() != Some(head.as_str())
9965                || !git::is_clean(&winner.worktree).await?
9966            {
9967                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9968                return Ok(false);
9969            }
9970        }
9971        let failure = if i >= self.state.config.graph.land_rounds
9972            || self.state.config.graph.review_rounds == 0
9973        {
9974            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9975        } else {
9976            None
9977        };
9978        if let Some(why) = failure {
9979            self.hold_merge_repair(i, why)?;
9980            return Ok(false);
9981        }
9982        if request.phase == Phase::Requested {
9983            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9984                Ok(pr) => pr,
9985                Err(e) => {
9986                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9987                    return Ok(false);
9988                }
9989            };
9990            self.state.merge_repairs[i].phase = Phase::Fixing;
9991            self.state.land_approval = None;
9992            self.state.land_armed_head = None;
9993            self.state.save()?;
9994            // Serialize the exact records, including finding ids/details,
9995            // reconsideration reasons, final votes and verification output.
9996            let evidence = serde_json::to_string_pretty(&request)?;
9997            let reason = format!(
9998                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9999            );
10000            let budget = self.state.config.graph.land_rounds;
10001            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
10002            match result {
10003                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
10004                Ok(land::Fixed::Deferred) => {
10005                    // An unavailable fixer did not spend this repair request.
10006                    // Restore partial fallback work before retrying it, as for
10007                    // an interrupted repair, without closing the request.
10008                    let worktree = self
10009                        .state
10010                        .winner()
10011                        .context("repair has no winning branch")?
10012                        .worktree
10013                        .clone();
10014                    self.graft_restore(&worktree, &request.head).await?;
10015                    self.state.merge_repairs[i].phase = Phase::Requested;
10016                    self.stop_for_resource()?;
10017                    return Ok(false);
10018                }
10019                Ok(land::Fixed::Failed(why)) => {
10020                    self.hold_merge_repair(
10021                        i,
10022                        format!(
10023                            "repair failed: {why}; inspect the branch and run an explicit review"
10024                        ),
10025                    )?;
10026                    return Ok(false);
10027                }
10028                Err(e) => {
10029                    self.hold_merge_repair(
10030                        i,
10031                        format!(
10032                            "repair failed: {e:#}; inspect the branch and run an explicit review"
10033                        ),
10034                    )?;
10035                    return Ok(false);
10036                }
10037            }
10038            // Even no change gets a real new reviewer wave. The previous
10039            // evidence remains in the request; a former clean/concluded round
10040            // cannot short-circuit the new review or gate.
10041            self.begin_merge_repair_review(i)?;
10042        }
10043        self.review_loop().await?;
10044        self.gate().await?;
10045        if self.resource_halt()? || self.state.parked {
10046            return Ok(false);
10047        }
10048        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10049            != Some(RunStatus::Gating)
10050            || !self.state.gate_allows_progress()
10051        {
10052            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10053            return Ok(false);
10054        }
10055        // Review and gate may have added commits after the initial repair.
10056        // Publish exactly their final head to the existing PR, with a lease.
10057        let winner = self
10058            .state
10059            .winner()
10060            .cloned()
10061            .context("repair has no winning branch")?;
10062        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10063        if self.state.gate_head.as_deref() != Some(after.as_str())
10064            || !git::is_clean(&winner.worktree).await?
10065        {
10066            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10067            return Ok(false);
10068        }
10069        let expected = self
10070            .state
10071            .pr_pushed_head
10072            .as_deref()
10073            .unwrap_or(&request.head)
10074            .to_owned();
10075        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10076        // Push the checked commit itself: a local ref moving after the check
10077        // must not change which commit the forge receives.
10078        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10079            return Ok(false);
10080        }
10081        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10082        let refspec = format!("{after}:refs/heads/{public}");
10083        let push = git::git_raw(
10084            &winner.worktree,
10085            &["push", &lease, &self.state.config.merge.remote, &refspec],
10086        )
10087        .await;
10088        match push {
10089            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10090            Ok(out) => {
10091                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10092                return Ok(false);
10093            }
10094            Err(e) => {
10095                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10096                return Ok(false);
10097            }
10098        }
10099        self.state.merge_repairs[i].phase = Phase::Complete;
10100        self.state.status = RunStatus::Landing;
10101        self.state.event(
10102            "repair",
10103            "new review and gate completed; fresh owner merge decision required",
10104        );
10105        self.state.save()?;
10106        Ok(true)
10107    }
10108
10109    fn merge_repair_artifacts(&self) -> PathBuf {
10110        let artifacts = agent::artifacts_dir(&self.state.dir());
10111        match self
10112            .state
10113            .merge_repairs
10114            .last()
10115            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10116        {
10117            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10118            None => artifacts,
10119        }
10120    }
10121
10122    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10123        if let Some(e2e) = self.state.head_e2e.take() {
10124            self.state
10125                .verification_history
10126                .push(crate::run::VerificationEvidence {
10127                    superseded_by: None,
10128                    archived_at: Some(Timestamp::now()),
10129                    fixes_spent: Some(self.state.gate_fixes.len()),
10130                    gate_head: None,
10131                    gate_ran: false,
10132                    gate: Vec::new(),
10133                    e2e: Some(e2e),
10134                });
10135        }
10136        self.state.reviews.clear();
10137        self.state.gate.clear();
10138        self.state.gate_ran = false;
10139        self.state.gate_head = None;
10140        self.state.gate_fixes.clear();
10141        self.state.gate_repair_e2e = None;
10142        self.state.pre_gate.clear();
10143        self.state.pre_gate_commit = None;
10144        self.state.contested_handoff = None;
10145        self.state.land_reruns = None;
10146        self.state.land_approval = None;
10147        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10148        self.state.status = RunStatus::Reviewing;
10149        self.state.save()
10150    }
10151
10152    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10153        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10154        self.state.merge_repairs[index].failure = Some(why.clone());
10155        self.state.status = RunStatus::Blocked;
10156        self.state.event("repair", why);
10157        self.state.save()
10158    }
10159
10160    // -------------------------------------------------------------- helpers
10161
10162    /// Fetch or create a seat, keeping its conversation across nodes.
10163    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10164        if let Some(existing) = self.state.seats.get(key)
10165            && existing.agent == agent
10166        {
10167            return existing.clone();
10168        }
10169        // A seat that changes agent mints its session id from the agent too,
10170        // like a handover: the old agent's uuid is already taken by the CLI.
10171        let fresh = if self.state.seats.contains_key(key) {
10172            handover_seat(key, agent, self.state.next_seat_seed())
10173        } else {
10174            SeatState::new(key, agent, self.state.seed)
10175        };
10176        self.state.seats.insert(key.to_owned(), fresh.clone());
10177        fresh
10178    }
10179
10180    /// The agent now holding seat `key`: `spec`, unless a handover moved the
10181    /// seat to another roster agent, in which case that agent. Nodes that
10182    /// continue a seat's conversation (deliberation, the votes, a reviewer's
10183    /// reconsideration) must keep talking to whoever answered it, not slip
10184    /// back to the agent that failed it.
10185    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10186        match self.state.seats.get(key) {
10187            Some(s) if s.agent != spec.id => {
10188                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10189            }
10190            _ => spec,
10191        }
10192    }
10193
10194    /// A candidate rendered for judging, with the leak policy applied.
10195    fn view(&self, c: &Candidate) -> CandidateView {
10196        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10197            .unwrap_or_default();
10198        let (patch, _) = blind::sanitize_patch(
10199            &format!("candidate {} patch", c.label),
10200            &raw,
10201            &self.state.config.blind,
10202        );
10203        CandidateView {
10204            label: c.label,
10205            branch: c.branch.clone(),
10206            summary: c.summary.clone(),
10207            stat: c.stat.clone(),
10208            patch,
10209        }
10210    }
10211
10212    /// The full candidate set as prompt text, for seats with no live session.
10213    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10214        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10215        prompt::judge(
10216            "(see above)",
10217            &views,
10218            self.roles.judges.len(),
10219            base_short,
10220            "en",
10221        )
10222    }
10223
10224    /// The final-vote prompt with everything a seat that has no session of its
10225    /// own needs: the candidates, the seat's own ranking and reasons, and the
10226    /// anonymised deliberation it took part in (only when there was one, so a
10227    /// handed-over seat never sees more than the seat it replaces did). The
10228    /// `Final vote` heading stays first.
10229    fn vote_prompt_full(
10230        &self,
10231        j: usize,
10232        labels: &[char],
10233        language: &str,
10234        candidates: &[Candidate],
10235        base_short: &str,
10236    ) -> String {
10237        let mut text = format!(
10238            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10239            prompt::final_vote(labels, language),
10240            self.state.instruction,
10241            self.candidate_block(candidates, base_short)
10242        );
10243        if let Some(own) = self
10244            .state
10245            .judgements
10246            .get(j)
10247            .filter(|r| !r.ranking.is_empty())
10248        {
10249            let reasons = own
10250                .reasons
10251                .iter()
10252                .map(|(k, v)| format!("- {k}: {v}"))
10253                .collect::<Vec<_>>()
10254                .join("\n");
10255            text.push_str(&format!(
10256                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10257                own.ranking.iter().collect::<String>(),
10258                if reasons.is_empty() {
10259                    ""
10260                } else {
10261                    ", because:\n"
10262                }
10263            ));
10264        }
10265        if !self.state.deliberation.is_empty() {
10266            text.push_str("\n# What was argued before this vote\n");
10267            for t in self.transcript(&[], j) {
10268                text.push_str(&format!(
10269                    "\n## {}{}\n\n{}\n",
10270                    t.who,
10271                    if t.is_self { " (you)" } else { "" },
10272                    t.body.trim()
10273                ));
10274            }
10275        }
10276        text
10277    }
10278
10279    /// Anonymised transcript for judge `self_idx`.
10280    ///
10281    /// The initial rankings are always the opening statements. Seeding them
10282    /// only when no turn had been taken yet meant every judge after the first
10283    /// argued against a single voice instead of against the actual split — the
10284    /// disagreement is the information, so it is always on the table.
10285    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10286        let mut turns = Vec::new();
10287        for j in &self.state.judgements {
10288            if j.ranking.is_empty() {
10289                continue;
10290            }
10291            let reasons = j
10292                .reasons
10293                .iter()
10294                .map(|(k, v)| format!("- {k}: {v}"))
10295                .collect::<Vec<_>>()
10296                .join("\n");
10297            turns.push(Turn {
10298                who: format!("Judge {} (opening ranking)", j.judge),
10299                is_self: j.judge == self_idx + 1,
10300                body: format!(
10301                    "Ranked {}{}{reasons}",
10302                    j.ranking.iter().collect::<String>(),
10303                    if reasons.is_empty() {
10304                        ""
10305                    } else {
10306                        ", because:\n"
10307                    }
10308                ),
10309            });
10310        }
10311        for t in self
10312            .state
10313            .deliberation
10314            .iter()
10315            .flat_map(|r| r.turns.iter())
10316            .chain(current)
10317        {
10318            turns.push(Turn {
10319                who: format!("Judge {}", t.judge),
10320                is_self: t.judge == self_idx + 1,
10321                body: t.body.clone(),
10322            });
10323        }
10324        turns
10325    }
10326}
10327
10328/// Does this seat still hold the context a follow-up prompt would rely on?
10329fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10330    agent::has_session(spec.kind, seat, sessions)
10331}
10332
10333/// The next entry in `roster` after `start`, never wrapping back to the
10334/// front, whose id is not in `tried` yet.
10335///
10336/// Starts one past `start` rather than at the front of `roster`: `start` is
10337/// the seat's own original position, and a seat whose candidate slot already
10338/// sits on the roster's second entry must fall through to the third next, not
10339/// restart at the first — which is very likely a different candidate's own
10340/// agent already. Never wraps back past `start`, for the same reason: an
10341/// entry earlier in the roster than the seat's own position is almost
10342/// certainly some *other* candidate slot's own agent, and once the tail of
10343/// the roster is exhausted there are no more untried agents for *this* seat
10344/// to fall through to — the caller's fallback chain ends there, exactly as
10345/// "no further untried agents remain in the list for that seat" asks for.
10346///
10347/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
10348/// the same id twice (an operator's `roles.implementers` typo, or a
10349/// `[[agents]]` list reused across roles) must not let
10350/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
10351/// over `roster` either finds an untried id or runs out, so this always
10352/// terminates regardless of duplicates.
10353fn next_untried_in_roster<'a>(
10354    roster: &'a [AgentSpec],
10355    start: usize,
10356    tried: &BTreeSet<String>,
10357) -> Option<&'a AgentSpec> {
10358    roster
10359        .get(start + 1..)?
10360        .iter()
10361        .find(|s| !tried.contains(&s.id))
10362}
10363
10364/// The successor for a seat, shared by all four seat kinds (implement, judge,
10365/// review, advise). `others` holds the ids that *currently* occupy the other
10366/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
10367/// sees them), so roster entries beyond the seat count act as spares: a failed
10368/// seat goes to an agent no other seat holds whenever the roster permits.
10369///
10370/// `carried` is `Some` only for the review loop's carried failure history.
10371/// Order: (1) forward from `start`, never wrapping, untried, not a carried
10372/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
10373/// whole roster: untried and not another seat's occupant; (3) the plain walk
10374/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
10375/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
10376/// a better outcome than an empty seat, and it keeps the answer to "is there
10377/// a successor at all" exactly what it was before occupants were considered,
10378/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
10379/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
10380/// carried failure is a cheaper bet than doubling an agent on the panel, and
10381/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
10382/// the same round are handled one at a time, so a seat later in the batch
10383/// sees an earlier one's new occupant but not yet its own freed agent.
10384fn pick_successor<'a>(
10385    roster: &'a [AgentSpec],
10386    start: usize,
10387    tried: &BTreeSet<String>,
10388    carried: Option<&BTreeSet<String>>,
10389    others: &BTreeSet<String>,
10390) -> Option<&'a AgentSpec> {
10391    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10392    roster
10393        .get(start + 1..)
10394        .and_then(|tail| {
10395            tail.iter()
10396                .filter(free)
10397                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10398        })
10399        .or_else(|| {
10400            carried
10401                .is_some()
10402                .then(|| roster.iter().find(free))
10403                .flatten()
10404        })
10405        .or_else(|| match carried {
10406            Some(c) => next_for_seat(roster, start, tried, c),
10407            None => next_untried_in_roster(roster, start, tried),
10408        })
10409}
10410
10411/// The next agent for a seat that carries its failure history across rounds
10412/// (the review loop). `round_tried` is this round's own bound and starts
10413/// empty every round; `carried_failed` only decides priority.
10414///
10415/// First: an id walking forward from `start`, never wrapping, that is neither
10416/// tried this round nor failed in an earlier one. Only when that is exhausted
10417/// does it rescue: the first roster id (in roster order, so this one *does*
10418/// look before `start`) not yet tried this round, which is by then a carried
10419/// failure. Each id is rescued at most once per round, so it cannot loop.
10420fn next_for_seat<'a>(
10421    roster: &'a [AgentSpec],
10422    start: usize,
10423    round_tried: &BTreeSet<String>,
10424    carried_failed: &BTreeSet<String>,
10425) -> Option<&'a AgentSpec> {
10426    roster
10427        .get(start + 1..)?
10428        .iter()
10429        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10430        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10431}
10432
10433/// Where a reviewer seat starts a round: the agent that last answered it when
10434/// it is still on the roster and not marked failed, else the spec's own agent
10435/// unless it failed, else the next roster agent that has not failed, else the
10436/// spec's own agent again (the whole roster failed). Ids no longer on the
10437/// roster are ignored. An empty roster has no handover, so the spec stands.
10438fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10439    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10440        return spec;
10441    };
10442    let ok = |id: &str| !h.failed.contains(id);
10443    if let Some(last) = h.last_ok.as_deref()
10444        && ok(last)
10445        && let Some(s) = roster.iter().find(|s| s.id == last)
10446    {
10447        return s.clone();
10448    }
10449    if ok(&spec.id) {
10450        return spec;
10451    }
10452    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10453    roster
10454        .get(from + 1..)
10455        .into_iter()
10456        .flatten()
10457        .chain(roster.iter())
10458        .find(|s| ok(&s.id))
10459        .cloned()
10460        .unwrap_or(spec)
10461}
10462
10463/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
10464/// seed so a CLI that mints its session id up front (`--session-id`) never
10465/// reuses the uuid the previous agent already opened under the same seat key.
10466pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10467    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10468}
10469
10470/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
10471/// so the classifier below and the code that builds the message cannot drift.
10472const TIMED_OUT: &str = "timed out";
10473
10474impl FailClass {
10475    /// `None` for an answer; otherwise how the turn failed.
10476    fn of(out: &AgentOutcome) -> Option<Self> {
10477        match out {
10478            AgentOutcome::Ok(_) => None,
10479            AgentOutcome::Quota(_) => Some(Self::Quota),
10480            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10481            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10482            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10483            // Not a failure of the agent. Callers check for it first; `None`
10484            // here means "no class to hand over on".
10485            AgentOutcome::ResourceBlocked(_) => None,
10486        }
10487    }
10488
10489    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10490    fn stem_word(&self) -> &'static str {
10491        match self {
10492            Self::Quota => "quota",
10493            _ => "handover",
10494        }
10495    }
10496}
10497
10498/// The message's first line with its variable parts removed — digit runs and
10499/// path-like tokens — so "exit code 2" and "exit code 7" read
10500/// as one kind of failure.
10501fn failure_signature(msg: &str) -> String {
10502    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10503    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10504    // the historical machine key while the process diagnosis uses readable prose.
10505    if let Some(code) = line
10506        .strip_prefix("process exited with no usable output (exit code ")
10507        .and_then(|s| s.strip_suffix(')'))
10508        .and_then(|s| s.parse::<i32>().ok())
10509    {
10510        line = format!("exited with some({code}) and no usable output");
10511    } else if matches!(
10512        line.as_str(),
10513        "process exited with no usable output (no exit code reported)"
10514            | "process exited with no usable output (exit code unavailable)"
10515    ) {
10516        line = "exited with none and no usable output".to_owned();
10517    }
10518    let mut out = Vec::new();
10519    for word in line.split_whitespace() {
10520        if word.contains('/') || word.contains('\\') {
10521            out.push("<path>".to_owned());
10522            continue;
10523        }
10524        let mut w = String::new();
10525        let mut in_digits = false;
10526        for c in word.chars() {
10527            if c.is_ascii_digit() {
10528                if !in_digits {
10529                    w.push('#');
10530                }
10531                in_digits = true;
10532            } else {
10533                in_digits = false;
10534                w.push(c);
10535            }
10536        }
10537        out.push(w);
10538    }
10539    out.join(" ").chars().take(120).collect()
10540}
10541
10542/// Whether a seat that just failed with `cur` may go to the next roster agent.
10543/// A quota or a timeout always may. Any other failure may not when the agent
10544/// before it failed the same way: an error the prompt causes would otherwise
10545/// walk the whole roster. `prev` is the class of the immediately preceding
10546/// agent's failure, so a quota or timeout in between breaks the run of
10547/// identical failures by itself.
10548fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10549    match cur {
10550        FailClass::Quota | FailClass::Timeout => true,
10551        FailClass::Other(_) => prev != Some(cur),
10552    }
10553}
10554
10555/// A short human reason for a failed outcome, for the handover record.
10556fn fail_reason(out: &AgentOutcome) -> String {
10557    match out {
10558        AgentOutcome::Ok(_) => String::new(),
10559        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10560        AgentOutcome::Dropped(o) => format!(
10561            "the CLI dropped the stream ({})",
10562            o.dropped
10563                .as_ref()
10564                .map(|d| d.why.as_str())
10565                .unwrap_or("it ended without delivering its answer")
10566        ),
10567        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10568            e.lines().next().unwrap_or("").chars().take(160).collect()
10569        }
10570    }
10571}
10572
10573/// Note one handover in the run: the structured record and, in the timeline,
10574/// the sentence a person reads. A quota keeps the wording it always had.
10575pub(crate) fn record_handover(
10576    state: &mut RunState,
10577    node: &str,
10578    seat: &str,
10579    from: &str,
10580    to: &str,
10581    class: &FailClass,
10582    reason: &str,
10583) {
10584    let message = if *class == FailClass::Quota {
10585        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10586    } else {
10587        format!("{seat}: handed over {from} -> {to} ({reason})")
10588    };
10589    state.event(node, message);
10590    state.handovers.push(Handover {
10591        at: Timestamp::now(),
10592        node: node.to_owned(),
10593        seat: seat.to_owned(),
10594        from: from.to_owned(),
10595        to: to.to_owned(),
10596        reason: reason.to_owned(),
10597    });
10598}
10599
10600/// Did this reply report running a command whose own CLI never confirmed an
10601/// exit status?
10602///
10603/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10604/// command *finished* (see that type's own doc), so this can only be `true`
10605/// for a command whose completion event carried no readable exit code — not
10606/// for one that simply is not mentioned at all. That is the one signal this
10607/// crate can read, from the same record `state.jobs` renders, about a reply
10608/// standing next to work its own CLI cannot vouch for finishing; it is
10609/// deliberately not a check on the exit code's *value* (a fixer legitimately
10610/// runs a command that fails mid-iteration before it succeeds) and not a
10611/// guess at a command still running in the background (which emits no event
10612/// at all, and so leaves no evidence here to find).
10613fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10614    commands.iter().any(|c| c.exit_code.is_none())
10615}
10616
10617/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10618/// trusted as a verified no-op — the adoption guard's own text-level half.
10619///
10620/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10621/// timed out): a marker only earns the benefit of the doubt from a turn the
10622/// CLI itself vouches for finishing properly, the same house style
10623/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10624/// *fix* report to for `commands`. A candidate that timed out, exited
10625/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10626/// is, whatever prose it wrote — this returns `None` before it ever looks at
10627/// `text`. The remaining guards (the tree really is empty, the evidence is
10628/// non-empty) are the caller's: this only reads what the reply *claimed*.
10629fn verified_noop_claim(
10630    usable: bool,
10631    commands: &[agent::CommandEvidence],
10632    text: &str,
10633) -> Option<String> {
10634    (usable && !has_unconfirmed_command(commands))
10635        .then(|| verdict::verified_noop(text))
10636        .flatten()
10637}
10638
10639fn short(commit: &str) -> String {
10640    commit.chars().take(7).collect()
10641}
10642
10643fn make_executable(path: &Path) -> Result<()> {
10644    #[cfg(unix)]
10645    {
10646        use std::os::unix::fs::PermissionsExt as _;
10647        let mut perms = std::fs::metadata(path)?.permissions();
10648        perms.set_mode(0o755);
10649        std::fs::set_permissions(path, perms)?;
10650    }
10651    #[cfg(not(unix))]
10652    {
10653        let _ = path;
10654    }
10655    Ok(())
10656}
10657
10658/// What every seat in one batch shares: where the answers are attributed, the
10659/// prompt overlay they inherit, and the build cache they are told to use.
10660///
10661/// A struct rather than four more parameters: `wave` also needs the run's
10662/// state (to record who is answering right now) and the attempt number, and
10663/// eight positional arguments is both unreadable and a clippy error.
10664struct WaveCtx<'a> {
10665    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10666    /// paid for it.
10667    run: &'a str,
10668    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10669    node: &'a str,
10670    prompts: &'a Prompts,
10671    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10672    cache: Option<&'a Path>,
10673    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10674    /// `JobRecord::round`. `None` for every other node.
10675    round: Option<usize>,
10676    /// Carry each seat's failed-agent history across waves (the review loop
10677    /// only): start-of-round priority and handover choice read
10678    /// [`RunState::seat_history`], and every answer or failure writes it.
10679    carry_seats: bool,
10680    /// The run's park / interrupt requests, so a wait for a shared resource
10681    /// can be abandoned the moment the operator asks to park.
10682    pauses: Option<(&'a Pause, &'a Pause)>,
10683}
10684
10685/// Shared scheduling policy for operations and spawned seat invocations.
10686fn repository_wait(
10687    state: &RunState,
10688    pauses: Option<(&Pause, &Pause)>,
10689) -> crate::repo_lock::WaitPolicy {
10690    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10691    let inherited = crate::repo_lock::WaitPolicy::current();
10692    crate::repo_lock::WaitPolicy::new(
10693        Duration::from_secs(state.config.graph.timeout_resource_wait),
10694        move || {
10695            inherited.cancelled()
10696                || pauses
10697                    .as_ref()
10698                    .is_some_and(|(a, b)| a.parked() || b.parked())
10699        },
10700    )
10701}
10702
10703fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10704    if error.is::<crate::repo_lock::ResourceBlocked>() {
10705        state.resource_block = Some(crate::run::ResourceBlock {
10706            cause: crate::run::ResourceBlockCause::Other,
10707            node: "repository".into(),
10708            kind: if error.to_string().contains("disk pressure") {
10709                "disk pressure"
10710            } else if error.to_string().contains("disk admission") {
10711                "disk admission"
10712            } else if error.to_string().contains(crate::incremental::BLOCKED) {
10713                "incremental cache admission"
10714            } else {
10715                "repository coordination"
10716            }
10717            .into(),
10718            detail: error.to_string(),
10719            at: Timestamp::now(),
10720        });
10721        state.event("resource", error.to_string());
10722    }
10723}
10724
10725async fn coordinate_operation(
10726    state: &mut RunState,
10727    cwd: &Path,
10728    wait: &crate::repo_lock::WaitPolicy,
10729) -> Result<Option<crate::repo_lock::SeatLease>> {
10730    let result =
10731        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10732            .await;
10733    if let Err(e) = &result {
10734        record_repository_block(state, e);
10735    }
10736    result
10737}
10738
10739/// Run one job, honouring the parallelism budget.
10740async fn run_one(
10741    job: SeatJob,
10742    sem: Arc<Semaphore>,
10743    ctx: &WaveCtx<'_>,
10744    state: &mut RunState,
10745    attempt: usize,
10746) -> (SeatState, AgentOutcome) {
10747    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10748        .await
10749        .pop()
10750        .expect("one job in, one result out");
10751    (seat, out)
10752}
10753
10754/// Run every job concurrently, capped by the semaphore, preserving order.
10755///
10756/// Every seat in the batch is recorded into [`RunState::active`] before the
10757/// wave starts and cleared as each answer lands, so the run's own record says
10758/// who is still being waited on rather than only who finished.
10759async fn wave(
10760    jobs: Vec<SeatJob>,
10761    sem: Arc<Semaphore>,
10762    ctx: &WaveCtx<'_>,
10763    state: &mut RunState,
10764    attempt: usize,
10765) -> Vec<(usize, SeatState, AgentOutcome)> {
10766    let WaveCtx {
10767        run,
10768        node,
10769        prompts,
10770        cache,
10771        round,
10772        carry_seats: _,
10773        pauses,
10774    } = *ctx;
10775    for job in &jobs {
10776        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10777    }
10778    if let Err(e) = state.save() {
10779        // A failed persist of "who is answering right now" must not abort the
10780        // wave: the seats are already being asked, and the alternative is
10781        // losing the answers to save a status line nobody may even be
10782        // watching.
10783        tracing::warn!("could not persist in-progress seats: {e:#}");
10784    }
10785    // Hold the shared build cache's lease for the whole batch, not per job:
10786    // several candidates (an implement wave) or a fixer legitimately share
10787    // one cache concurrently within this run, and that stays untouched — a
10788    // single lease taken once for the whole wave and released once it is
10789    // done is what stops a *different* borrower (another run's own wave, its
10790    // e2e/gate, a human's `magi review`) from interleaving a build into the
10791    // same directory while this one is in flight. Best-effort, not
10792    // all-or-nothing: a wave that cannot get the lease within its own
10793    // longest job's budget still runs — an hour of paid implementer calls is
10794    // not thrown away over cache contention — but every write-allowed seat
10795    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10796    // below), the same fallback a read-only seat always gets, rather than
10797    // building into a directory this run was never granted. The identity
10798    // record is still invalidated below either way, so the next tracked
10799    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10800    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10801    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10802    // never borrowed from a job's agent budget (and never charged to it either,
10803    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10804    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10805    let cache_guard = if let Some(cache_dir) = cache {
10806        if jobs_had_a_writer {
10807            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10808            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10809            match acquire_cache_lease(
10810                state,
10811                cache_dir,
10812                &owner,
10813                resource_limit,
10814                node,
10815                node,
10816                &cancel,
10817            )
10818            .await
10819            {
10820                Ok(Some(g)) => Some(g),
10821                // Parking: leave without spawning anything. The run parks at
10822                // the next boundary and the seats are asked afresh on resume.
10823                Ok(None) => {
10824                    if let Some(block) = &mut state.resource_block {
10825                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10826                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10827                        };
10828                    }
10829                    return block_wave(
10830                        jobs,
10831                        state,
10832                        node,
10833                        attempt,
10834                        "parked while waiting for the shared build cache",
10835                    );
10836                }
10837                // An unreadable lease is not contention: keep the old
10838                // fallback (no shared cache for this wave's seats).
10839                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10840                Err(e) => {
10841                    // Writing seats may not build into a cache this run does
10842                    // not hold, and an agent that cannot be given one is not
10843                    // asked at all: the run stops resumably instead.
10844                    let why = format!("{e:#}");
10845                    state.event(
10846                        "resource",
10847                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10848                    );
10849                    return block_wave(jobs, state, node, attempt, &why);
10850                }
10851            }
10852        } else {
10853            None
10854        }
10855    } else {
10856        None
10857    };
10858    // Every job keeps its whole budget: the wait above was not the agent's.
10859    // Restart the seats' clocks so "started" means "being spawned now".
10860    state.seats_spawning(node, attempt);
10861    if let Err(e) = state.save() {
10862        tracing::warn!("could not persist the start of a wave: {e:#}");
10863    }
10864    // Queueing behind this wave's own seats for a parallelism slot is normal
10865    // (each of them is bounded by its own agent timeout); only a wait beyond
10866    // all of them plus the resource limit means the slots never came back.
10867    let slot_limit = resource_limit
10868        + jobs
10869            .iter()
10870            .map(|j| j.timeout)
10871            .max()
10872            .unwrap_or_default()
10873            .saturating_mul(jobs.len() as u32);
10874    let mut set = tokio::task::JoinSet::new();
10875    // Seats announce themselves here the moment they hold a parallelism
10876    // permit, i.e. just before their CLI is spawned.
10877    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10878    let overlay = prompts.overlay(node);
10879    let repository_wait = repository_wait(state, pauses);
10880    for (i, mut job) in jobs.into_iter().enumerate() {
10881        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10882        if cache.is_some() {
10883            job.prompt.push('\n');
10884            job.prompt
10885                .push_str(&prompt::build_cache_note(node, job.allow_write));
10886        }
10887        let sem = Arc::clone(&sem);
10888        let run = run.to_owned();
10889        let node = node.to_owned();
10890        // Only implementers were told about the task's attachments, so only
10891        // their seats get the directory widened for reading.
10892        let attachments = if node == "implement" {
10893            state.attachments.clone()
10894        } else {
10895            Vec::new()
10896        };
10897        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10898        // `prompt::build_cache_note`'s doc for why setting it anyway is
10899        // exactly how a sandboxed reviewer's write refusal got reported as a
10900        // defect in the patch, not a property of its own seat. And a
10901        // write-allowed one is handed it only when the lease above was
10902        // actually acquired: a wave that could not get it (`cache_guard` is
10903        // `None`, see its own comment) must not send seats to build into a
10904        // directory this run does not hold - that is the exact concurrent,
10905        // unmanaged-write race this module exists to prevent, not something
10906        // "proceeding anyway" is allowed to reintroduce.
10907        let cache = cache
10908            .filter(|_| job.allow_write && cache_guard.is_some())
10909            .map(Path::to_path_buf);
10910        let spawned_tx = spawned_tx.clone();
10911        let repository_wait = repository_wait.clone();
10912        let coordination = state.config.coordination.clone();
10913        let fetched_base = state.fetched_base.clone();
10914        set.spawn(async move {
10915            // A wait for a parallelism slot is scheduling, not the agent: it
10916            // is bounded by its own limit and ends as a resource block, never
10917            // as a timeout of the agent that was not started.
10918            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10919                Ok(p) => p,
10920                Err(_) => {
10921                    return (
10922                        i,
10923                        job.seat,
10924                        AgentOutcome::ResourceBlocked(format!(
10925                            "no parallel slot became free within {}s",
10926                            slot_limit.as_secs()
10927                        )),
10928                        false,
10929                    );
10930                }
10931            };
10932            spawned_tx.send(job.seat.key.clone()).ok();
10933            let mut seat = job.seat;
10934            let out = repository_wait
10935                .scope(agent::invoke(
10936                    &job.spec,
10937                    &mut seat,
10938                    &Invocation {
10939                        coordination: coordination.as_ref(),
10940                        fetched_base: fetched_base.as_ref(),
10941                        cwd: &job.cwd,
10942                        prompt: &job.prompt,
10943                        timeout: job.timeout,
10944                        allow_write: job.allow_write,
10945                        sessions: job.sessions,
10946                        artifacts: &job.artifacts,
10947                        stem: &job.stem,
10948                        run: &run,
10949                        node: &node,
10950                        cache_dir: cache.as_deref(),
10951                        attachments: &attachments,
10952                        writable: &[],
10953                    },
10954                ))
10955                .await;
10956            let interrupted = out
10957                .as_ref()
10958                .err()
10959                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10960            let out = match out {
10961                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10962                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10963                // Billed work the CLI failed to hand over is not an ordinary
10964                // failure, but its text is the CLI's raw error JSON, not an
10965                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10966                // read it as one by forgetting to check. `usable()` is always
10967                // false here (dropped implies an empty response), so this has
10968                // to be checked before the catch-all `Failed` below or the
10969                // one shape this exists for is lost with the rest.
10970                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10971                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10972                Ok(o) => AgentOutcome::Failed(format!(
10973                    "process exited with no usable output ({})",
10974                    crate::notices::exit_status(o.exit_code)
10975                )),
10976                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10977                    AgentOutcome::ResourceBlocked(e.to_string())
10978                }
10979                Err(e) => AgentOutcome::Failed(e.to_string()),
10980            };
10981            (i, seat, out, interrupted)
10982        });
10983    }
10984    drop(spawned_tx);
10985    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10986    loop {
10987        let joined = tokio::select! {
10988            Some(key) = spawned_rx.recv() => {
10989                // The clock of this seat starts now, not when it queued.
10990                state.seat_spawned(&key);
10991                if let Err(e) = state.save() {
10992                    tracing::warn!("could not persist a seat's start: {e:#}");
10993                }
10994                continue;
10995            }
10996            joined = set.join_next() => joined,
10997        };
10998        let Some(joined) = joined else {
10999            break;
11000        };
11001        let (i, seat, out, interrupted) = match joined {
11002            Ok(v) => v,
11003            // No seat to clear: a panicked task never reported which one it
11004            // was. The defensive sweep below this loop is what stops that
11005            // seat's `active` entry from surviving forever.
11006            Err(e) => {
11007                tracing::error!("agent task panicked: {e}");
11008                continue;
11009            }
11010        };
11011        state.seat_finished(&seat.key);
11012        if interrupted
11013            && node == "implement"
11014            && let Some(label) = state
11015                .candidates
11016                .iter()
11017                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11018                .map(|candidate| candidate.label)
11019        {
11020            state.candidate_preservation.insert(
11021                label,
11022                "interrupted implementation awaiting persistence".into(),
11023            );
11024            state.interrupted_implementations.insert(label);
11025        }
11026        if let AgentOutcome::ResourceBlocked(why) = &out
11027            && state.resource_block.is_none()
11028        {
11029            state.resource_block = Some(crate::run::ResourceBlock {
11030                cause: crate::run::ResourceBlockCause::Other,
11031                node: node.to_owned(),
11032                kind: if why.contains("disk pressure") {
11033                    "disk pressure"
11034                } else if why.contains("disk admission") {
11035                    "disk admission"
11036                } else if why.contains(crate::incremental::BLOCKED) {
11037                    "incremental cache admission"
11038                } else if why.contains("repository coordination") {
11039                    "repository coordination"
11040                } else {
11041                    "parallel slot"
11042                }
11043                .to_owned(),
11044                detail: why.clone(),
11045                at: Timestamp::now(),
11046            });
11047            let activity = if interrupted {
11048                "agent execution interrupted"
11049            } else {
11050                "no agent was started"
11051            };
11052            state.event("resource", format!("{node}: {activity} - {why}"));
11053        }
11054        record_jobs(state, node, round, &seat.key, &out);
11055        if let Err(e) = state.save() {
11056            tracing::warn!("could not persist a seat's completion: {e:#}");
11057        }
11058        if collected.len() <= i {
11059            collected.resize_with(i + 1, || None);
11060        }
11061        collected[i] = Some((i, seat, out));
11062    }
11063    // Belt-and-braces for the panic branch above: every seat this exact batch
11064    // started shares this `(node, attempt)` pair, and every seat that finished
11065    // normally already cleared itself, so anything left tagged with it here
11066    // can only be a panicked task's leftover. Cleared unconditionally rather
11067    // than left to read as still answering forever.
11068    if state
11069        .active
11070        .values()
11071        .any(|a| a.node == node && a.attempt == attempt)
11072    {
11073        state
11074            .active
11075            .retain(|_, a| !(a.node == node && a.attempt == attempt));
11076        if let Err(e) = state.save() {
11077            tracing::warn!("could not persist the end of a wave: {e:#}");
11078        }
11079    }
11080    // Whether or not the lease above was actually held, several worktrees
11081    // may just have built into the cache with nothing here able to name one
11082    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
11083    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
11084    // clean it might not have strictly needed; trusting a stale match would
11085    // cost it a wrong answer.
11086    if let Some(cache_dir) = cache
11087        && jobs_had_a_writer
11088    {
11089        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11090    }
11091    if let Some(guard) = cache_guard {
11092        guard.release();
11093    }
11094    collected.into_iter().flatten().collect()
11095}
11096
11097/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
11098/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
11099/// this is data collection, not the fix-specific completion contract in
11100/// [`Runner::continue_fix_report`], and applies regardless of which node
11101/// asked.
11102///
11103/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
11104/// evidence from; `Failed` does not, and correctly contributes nothing — a
11105/// timeout or crash is not itself evidence about a command the seat may have
11106/// started.
11107fn record_jobs(
11108    state: &mut RunState,
11109    node: &str,
11110    round: Option<usize>,
11111    seat: &str,
11112    out: &AgentOutcome,
11113) {
11114    let commands: &[agent::CommandEvidence] = match out {
11115        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11116        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11117    };
11118    let checked_at = Timestamp::now();
11119    for c in commands {
11120        state.jobs.push(JobRecord {
11121            node: node.to_owned(),
11122            round,
11123            seat: seat.to_owned(),
11124            id: c.id.clone(),
11125            description: c.description.clone(),
11126            checked_at,
11127            status: match c.exit_code {
11128                Some(0) => JobStatus::Completed,
11129                Some(_) => JobStatus::Failed,
11130                None => JobStatus::Unknown,
11131            },
11132            exit_code: c.exit_code,
11133            result_summary: c.result_summary.clone(),
11134            source: c.source.clone(),
11135        });
11136    }
11137}
11138
11139/// Is a review round clean, given how many reviewer seats answered against
11140/// how many the round expected?
11141///
11142/// A seat that never answered (timeout, crash, unparsable output) is not a
11143/// seat that read the patch and found nothing — treating it as such is
11144/// exactly the bug this function exists to close. Under the default `block`
11145/// policy a missing seat can never be clean; `warn` still requires the seats
11146/// that *did* answer to have found nothing blocking and verification to be
11147/// green.
11148///
11149/// `quota_missing` narrows that `block` default for exactly one cause of
11150/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
11151/// a session limit lifts by the very next round buys nothing — the seat is
11152/// asked again with the same quota — so once every missing seat is accounted
11153/// for by a quota loss (and at least one seat *did* answer, so a decision has
11154/// something to rest on) the round is decided on the panel that could answer,
11155/// same as `warn` would. A panel that lost every seat to quota is not
11156/// decided here: `answered == 0` falls through to the existing `block`
11157/// fallback so a fully collapsed panel still waits rather than landing on no
11158/// review at all.
11159fn round_is_clean(
11160    blocking: usize,
11161    e2e_ok: bool,
11162    answered: usize,
11163    expected: usize,
11164    quota_missing: usize,
11165    policy: IncompleteReviewPolicy,
11166) -> bool {
11167    if blocking != 0 || !e2e_ok {
11168        return false;
11169    }
11170    if answered == expected || policy == IncompleteReviewPolicy::Warn {
11171        return true;
11172    }
11173    answered > 0 && expected - answered <= quota_missing
11174}
11175
11176/// The review loop's own conclusion, derived entirely from its persisted
11177/// round records and the round budget that produced them — never from
11178/// `status`, so a reentry (or `gate`/`merge` reading it independently)
11179/// recomputes the identical answer regardless of what an earlier node in the
11180/// same walk, or a previous walk, did to `status`.
11181///
11182/// `None` while more rounds remain to try, including when review never ran
11183/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
11184/// gone clean, or the budget is spent, or the tree has stopped moving (see
11185/// [`STAGNANT_LIMIT`]), the answer is one of two things:
11186///
11187/// - An incomplete panel that raised nothing is missing input, not a
11188///   verified tree — never a hand-off candidate, whatever verification said
11189///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
11190/// - Otherwise, green e2e on the last round hands off (see
11191///   [`Runner::stop_reviewing`]); red e2e blocks.
11192///
11193/// A last round whose own verification is still `ResourceBlocked` — magi
11194/// itself never got a command to run, not evidence the patch is broken —
11195/// is neither: this returns `None` for it too, the same as "more rounds
11196/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
11197/// handling of that shape) instead of this cheap recomputation guessing a
11198/// verdict a real attempt never produced.
11199pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11200    if max_rounds == 0
11201        || reviews.last().is_some_and(|r| {
11202            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11203        })
11204    {
11205        return Some(RunStatus::Gating);
11206    }
11207    let last = reviews.last()?;
11208    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11209    if reviews.len() < max_rounds && !stagnant {
11210        return None;
11211    }
11212    if last.incomplete() && last.blocking == 0 {
11213        return Some(RunStatus::Blocked);
11214    }
11215    if last.e2e_status() == E2eStatus::ResourceBlocked
11216        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11217    {
11218        return None;
11219    }
11220    Some(if last.e2e.iter().all(CommandOutcome::ok) {
11221        RunStatus::Gating
11222    } else {
11223        RunStatus::Blocked
11224    })
11225}
11226
11227/// How long a re-ask may take, given the budget the first attempt had.
11228///
11229/// A `nudged` retry is a request to restate an answer the seat has already
11230/// worked out: it carries no new work, so it does not deserve the original
11231/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
11232/// 133 seconds while a third sat for over ten minutes on a resumed session
11233/// holding 410 KB of prior output - and because the retry had inherited the
11234/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
11235/// judging round whose other seats were long finished.
11236///
11237/// A quarter of the budget, with a floor so that a deliberately short timeout
11238/// does not collapse to nothing. A retry that re-sends the whole prompt
11239/// (because the seat kept no context) is the original job again, and keeps the
11240/// original budget.
11241pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11242    if nudged {
11243        (full / 4).max(Duration::from_secs(120)).min(full)
11244    } else {
11245        full
11246    }
11247}
11248
11249/// Run a wave and parse each reply, re-asking the seats whose reply was
11250/// unusable.
11251///
11252/// The re-ask is a nudge rather than the whole prompt again when the seat still
11253/// holds its conversation, which is the difference between a cheap retry and
11254/// paying for the entire candidate set twice.
11255///
11256/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
11257/// successor in `roster` is handed to it instead of being re-asked: the
11258/// handover is the retry. A seat with no successor left (a single-agent
11259/// roster, the roster's tail) is nudged as before, up to `retries` times. So
11260/// the asks to one seat in one node number at most
11261/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
11262/// is asked once (a dropped stream is nudged first), and only the last agent
11263/// of the chain gets the `retries` same-agent nudges. Each roster agent is
11264/// tried at most once per seat, walking forward from the seat's own position and never wrapping
11265/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
11266/// other failure stops the chain when the previous agent failed the same way
11267/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
11268/// [`has_context`] is false and the job's own full prompt and full budget are
11269/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
11270/// intermediate ones are not losses) and is returned as a failure like any
11271/// other absent seat — the caller decides whether the panel still has a
11272/// quorum. An empty `roster` disables handover: failures are nudged as they
11273/// always were, and a quota is simply lost. A reply that fails to parse or
11274/// validate is the prompt's doing and is only ever nudged, never handed over.
11275///
11276/// The returned [`SeatState`] names the agent that answered (or tried last).
11277#[allow(clippy::too_many_arguments)]
11278async fn ask_json_wave<T>(
11279    jobs: Vec<SeatJob>,
11280    sem: Arc<Semaphore>,
11281    retries: usize,
11282    roster: &[AgentSpec],
11283    ctx: &WaveCtx<'_>,
11284    losses: &mut Vec<QuotaLoss>,
11285    state: &mut RunState,
11286    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11287) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11288where
11289    T: serde::de::DeserializeOwned + Send + 'static,
11290{
11291    ask_wave_with(
11292        jobs,
11293        sem,
11294        retries,
11295        roster,
11296        ctx,
11297        losses,
11298        state,
11299        &|text: &str| {
11300            let v = verdict::extract_json::<T>(text)?;
11301            validate(&v)?;
11302            Ok(v)
11303        },
11304    )
11305    .await
11306}
11307
11308/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
11309/// a node whose answer is prose (deliberation) shares the same handover,
11310/// failure classification, quota bookkeeping and bounds instead of a copy.
11311///
11312/// A seat handed to another roster agent is sent the job's `handover` prompt
11313/// (when it has one) rather than `prompt`: the new agent has no session, so a
11314/// resume-style prompt would be incomplete. That holds for the handover ask
11315/// and for every nudge to that agent whose `has_context` is false.
11316#[allow(clippy::too_many_arguments)]
11317async fn ask_wave_with<T>(
11318    jobs: Vec<SeatJob>,
11319    sem: Arc<Semaphore>,
11320    retries: usize,
11321    roster: &[AgentSpec],
11322    ctx: &WaveCtx<'_>,
11323    losses: &mut Vec<QuotaLoss>,
11324    state: &mut RunState,
11325    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11326) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11327where
11328    T: Send + 'static,
11329{
11330    let n = jobs.len();
11331    let originals: Vec<SeatJob> = jobs;
11332    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11333    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11334    // Nudges each seat's *current* agent has taken — 0 for a first-ask
11335    // answer, N once it has gone through N nudges. Read back once this
11336    // returns, so a caller building a history record (`ReviewRecord`) can
11337    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
11338    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
11339    // field's own doc.
11340    let mut nudges: Vec<usize> = vec![0; n];
11341    // The agent now occupying each seat, the ids it has already been through,
11342    // where in the roster the walk began, the class of the last failure, and
11343    // the stem word of a handover not yet asked (full prompt, full budget).
11344    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11345    let mut tried: Vec<BTreeSet<String>> = specs
11346        .iter()
11347        .map(|s| BTreeSet::from([s.id.clone()]))
11348        .collect();
11349    let starts: Vec<usize> = specs
11350        .iter()
11351        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11352        .collect();
11353    let carry = ctx.carry_seats && !roster.is_empty();
11354    // Carried across rounds: ids that failed the seat earlier, and how the
11355    // last failure went (so a repeat of it is not handed over again).
11356    let carried: Vec<BTreeSet<String>> = originals
11357        .iter()
11358        .map(|j| {
11359            state
11360                .seat_history
11361                .get(&j.seat.key)
11362                .filter(|_| carry)
11363                .map(|h| h.failed.clone())
11364                .unwrap_or_default()
11365        })
11366        .collect();
11367    let mut prev: Vec<Option<FailClass>> = originals
11368        .iter()
11369        .map(|j| {
11370            state
11371                .seat_history
11372                .get(&j.seat.key)
11373                .filter(|_| carry)
11374                .and_then(|h| h.last_fail.clone())
11375        })
11376        .collect();
11377    let next_agent =
11378        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11379            let others: BTreeSet<String> = specs
11380                .iter()
11381                .enumerate()
11382                .filter(|(j, _)| *j != i)
11383                .map(|(_, s)| s.id.clone())
11384                .collect();
11385            pick_successor(
11386                roster,
11387                starts[i],
11388                tried,
11389                carry.then(|| &carried[i]),
11390                &others,
11391            )
11392            .cloned()
11393        };
11394    let mut fresh: Vec<Option<String>> = vec![None; n];
11395    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11396    let mut pending: Vec<usize> = (0..n).collect();
11397
11398    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
11399    // asks: an agent with a successor is asked once and handed over, and only
11400    // a seat with no successor spends `retries` nudges on the same agent. This
11401    // only guarantees the loop's own termination whatever those say.
11402    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11403    for round in 0..max_rounds {
11404        if pending.is_empty() {
11405            break;
11406        }
11407        let mut batch = Vec::with_capacity(pending.len());
11408        let mut renudged: Vec<&str> = Vec::new();
11409        for &i in &pending {
11410            let src = &originals[i];
11411            // A seat now held by another agent than the job named has no
11412            // session of its own: it gets the full-context prompt whenever
11413            // it is asked in full (the handover ask, a nudge it cannot
11414            // resume).
11415            let full: &str = match &src.handover {
11416                Some(h) if specs[i].id != src.spec.id => h,
11417                _ => &src.prompt,
11418            };
11419            // The prompt and the budget are one decision: a nudge restates
11420            // finished work, a re-sent prompt redoes it.
11421            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11422                (
11423                    full.to_owned(),
11424                    src.timeout,
11425                    format!("{}-{word}-{}", src.stem, specs[i].id),
11426                )
11427            } else if nudges[i] == 0 {
11428                (full.to_owned(), src.timeout, src.stem.clone())
11429            } else {
11430                renudged.push(src.seat.key.as_str());
11431                let why = done[i]
11432                    .as_ref()
11433                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
11434                    .unwrap_or_else(|| "no parsable answer".to_owned());
11435                let nudge = prompt::nudge(&why);
11436                let nudged = has_context(&specs[i], &seats[i], src.sessions);
11437                let prompt = if nudged {
11438                    nudge
11439                } else {
11440                    format!("{full}\n\n---\n\n{nudge}")
11441                };
11442                (
11443                    prompt,
11444                    retry_budget(src.timeout, nudged),
11445                    format!("{}-retry{}", src.stem, nudges[i]),
11446                )
11447            };
11448            batch.push(SeatJob {
11449                spec: specs[i].clone(),
11450                seat: seats[i].clone(),
11451                cwd: src.cwd.clone(),
11452                prompt,
11453                timeout,
11454                allow_write: src.allow_write,
11455                sessions: src.sessions,
11456                artifacts: src.artifacts.clone(),
11457                stem,
11458                handover: None,
11459            });
11460        }
11461
11462        if !renudged.is_empty() {
11463            state.event(
11464                ctx.node,
11465                format!("retry {round}: re-asking {}", renudged.join(", ")),
11466            );
11467        }
11468        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11469        let mut still = Vec::new();
11470        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11471            seats[i] = seat;
11472            // No agent was asked: nothing to hand over, nudge, remember or
11473            // charge. The caller sees `resource_block` and stops the node.
11474            if let AgentOutcome::ResourceBlocked(why) = &out {
11475                done[i] = Some(Err(anyhow::anyhow!("{why}")));
11476                continue;
11477            }
11478            let class = FailClass::of(&out);
11479            // A dropped stream is nudged first (the conversation is still
11480            // there to pick up); only a seat whose nudges are spent hands over.
11481            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11482                && nudges[i] < retries
11483                && !roster.is_empty();
11484            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11485                let next = should_hand_over(prev[i].as_ref(), &cur)
11486                    .then(|| next_agent(i, &tried[i], &specs))
11487                    .flatten();
11488                if carry {
11489                    let h = state
11490                        .seat_history
11491                        .entry(originals[i].seat.key.clone())
11492                        .or_default();
11493                    h.failed.insert(specs[i].id.clone());
11494                    h.last_fail = Some(cur.clone());
11495                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11496                        h.last_ok = None;
11497                    }
11498                    // Saved before the next agent is asked, so a restart in
11499                    // between does not forget who failed.
11500                    if let Err(e) = state.save() {
11501                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11502                    }
11503                }
11504                if let Some(next) = next {
11505                    record_handover(
11506                        state,
11507                        ctx.node,
11508                        &originals[i].seat.key,
11509                        &specs[i].id,
11510                        &next.id,
11511                        &cur,
11512                        &fail_reason(&out),
11513                    );
11514                    tried[i].insert(next.id.clone());
11515                    prev[i] = Some(cur.clone());
11516                    seats[i] =
11517                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11518                    specs[i] = next;
11519                    fresh[i] = Some(cur.stem_word().to_owned());
11520                    nudges[i] = 0;
11521                    done[i] = Some(Err(anyhow::anyhow!(
11522                        "handed over after: {}",
11523                        fail_reason(&out)
11524                    )));
11525                    still.push(i);
11526                    continue;
11527                }
11528            }
11529            if carry
11530                && !nudge_first
11531                && let Some(cur) = class.clone()
11532            {
11533                // The chain ended here (no successor, or a repeated failure).
11534                let h = state
11535                    .seat_history
11536                    .entry(originals[i].seat.key.clone())
11537                    .or_default();
11538                h.failed.insert(specs[i].id.clone());
11539                h.last_fail = Some(cur);
11540            }
11541            let parsed = match out {
11542                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11543                AgentOutcome::Quota(o) => {
11544                    last_quota[i] = o.quota.clone();
11545                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11546                }
11547                // Not a parseable answer: the nudge loop re-asks it, which is
11548                // exactly what a dropped stream needs. Just don't hand its raw
11549                // error JSON to `extract_json`.
11550                AgentOutcome::Dropped(o) => {
11551                    let why = o
11552                        .dropped
11553                        .as_ref()
11554                        .map(|d| d.why.as_str())
11555                        .unwrap_or("the CLI ended the stream without delivering its answer");
11556                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11557                }
11558                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11559                    Err(anyhow::anyhow!(e))
11560                }
11561            };
11562            let quota = class == Some(FailClass::Quota);
11563            let failed = parsed.is_err();
11564            if !failed && matches!(ctx.node, "judge" | "vote") {
11565                // A validated answer restores this seat's quorum presence,
11566                // including on the normal path after an all-seat block.
11567                state
11568                    .quota
11569                    .retain(|loss| loss.seat != originals[i].seat.key);
11570            }
11571            done[i] = Some(parsed);
11572            if carry && !failed {
11573                let h = state
11574                    .seat_history
11575                    .entry(originals[i].seat.key.clone())
11576                    .or_default();
11577                h.failed.remove(&specs[i].id);
11578                h.last_ok = Some(specs[i].id.clone());
11579                h.last_fail = None;
11580            }
11581            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11582            // fail the same way; and never re-ask a seat that already parsed.
11583            // A failed agent that still has a successor is not re-asked
11584            // either: the handover was its remedy and has just been refused
11585            // (the chain stops on a repeated failure class). A seat with no
11586            // successor left (a single-agent roster, the roster's tail, or an
11587            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11588            let agent_failure = class.is_some()
11589                && !nudge_first
11590                && !roster.is_empty()
11591                && next_agent(i, &tried[i], &specs).is_some();
11592            if failed && !quota && !agent_failure && nudges[i] < retries {
11593                nudges[i] += 1;
11594                still.push(i);
11595            }
11596        }
11597        pending = still;
11598    }
11599
11600    // One loss per seat whose chain ended on a quota: the intermediate ones
11601    // were absorbed by a handover and are not losses.
11602    for (i, q) in last_quota.into_iter().enumerate() {
11603        if let Some(quota) = q {
11604            losses.push(QuotaLoss::from_report(
11605                originals[i].seat.key.clone(),
11606                ctx.node,
11607                Some(&quota),
11608            ));
11609        }
11610    }
11611
11612    // A stalled quorum already has its own recovery checkpoint. Refresh
11613    // its missing seats before deciding whether it can resume.
11614    if state.status != RunStatus::Stalled
11615        && ctx.node != "deliberate"
11616        && n > 0
11617        && losses.len() == n
11618        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11619    {
11620        block_machine_outage(state, ctx.node, losses);
11621        state.quota.append(losses);
11622    }
11623
11624    seats
11625        .into_iter()
11626        .zip(done)
11627        .zip(nudges)
11628        .map(|((seat, res), attempts)| {
11629            (
11630                seat,
11631                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11632                attempts,
11633            )
11634        })
11635        .collect()
11636}
11637
11638/// A node with no available agent has no task verdict. Preserve the node
11639/// checkpoint and record why a subsequent explicit resume is needed.
11640pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11641    let mut auth: Vec<_> = losses
11642        .iter()
11643        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11644        .map(|loss| loss.agent.clone())
11645        .collect();
11646    auth.sort();
11647    auth.dedup();
11648    let (kind, detail) = if auth.is_empty() {
11649        (
11650            "agent usage limit",
11651            "All seats reached their agent usage limits; resume after usage is available"
11652                .to_owned(),
11653        )
11654    } else {
11655        (
11656            "agent authentication",
11657            format!(
11658                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11659                auth.join(", ")
11660            ),
11661        )
11662    };
11663    state.resource_block = Some(crate::run::ResourceBlock {
11664        node: node.to_owned(),
11665        kind: kind.to_owned(),
11666        cause: crate::run::ResourceBlockCause::Other,
11667        detail: detail.clone(),
11668        at: Timestamp::now(),
11669    });
11670    state.event(node, detail);
11671}
11672
11673/// Did the wave end without asking any agent (see [`block_wave`])?
11674fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11675    results
11676        .iter()
11677        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11678}
11679
11680/// Settle a wave that never spawned anything: every job answers
11681/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11682/// else about the seat (session, history, quota) is touched.
11683fn block_wave(
11684    jobs: Vec<SeatJob>,
11685    state: &mut RunState,
11686    node: &str,
11687    attempt: usize,
11688    why: &str,
11689) -> Vec<(usize, SeatState, AgentOutcome)> {
11690    state.resource_wait = None;
11691    state
11692        .active
11693        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11694    if let Err(e) = state.save() {
11695        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11696    }
11697    jobs.into_iter()
11698        .enumerate()
11699        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11700        .collect()
11701}
11702
11703/// Acquire the shared build cache's lease, waiting out contention within
11704/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11705/// unbounded wait is never acceptable).
11706///
11707/// A first, non-blocking check happens before ever waiting; if it finds the
11708/// lease busy, that fact is logged as a `verify` event *and* flushed with
11709/// [`RunState::save`] immediately — not only once the wait finally succeeds
11710/// or gives up — so a `magi show` run by a different process while this one
11711/// is still waiting reads a `run.json` that says so, rather than whatever it
11712/// looked like before the wait started. The same applies to the terminal
11713/// failure: logged and saved before this returns `Err`, so a caller that
11714/// could not get the lease at all still leaves a legible record of why.
11715async fn acquire_cache_lease(
11716    state: &mut RunState,
11717    cache_dir: &Path,
11718    owner: &crate::cache::Owner,
11719    budget: Duration,
11720    node: &str,
11721    context: &str,
11722    cancel: &(dyn Fn() -> bool + Sync),
11723) -> Result<Option<crate::cache::Guard>> {
11724    let home = crate::run::home();
11725    let started = Instant::now();
11726    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11727        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11728        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11729        Err(e) => {
11730            state.event(
11731                "verify",
11732                format!("{context}: could not check the shared build cache: {e:#}"),
11733            );
11734            if let Err(e2) = state.save() {
11735                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11736            }
11737            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11738        }
11739    };
11740    state.event(
11741        "verify",
11742        format!(
11743            "{context}: waiting for the shared build cache at {} ({})",
11744            cache_dir.display(),
11745            busy.describe()
11746        ),
11747    );
11748    // Seats of this node are waiting, not running: say so, so the countdown a
11749    // reader sees only ever measures an agent that has been spawned.
11750    state.resource_wait = Some(crate::run::ResourceWait {
11751        node: context.to_owned(),
11752        kind: "shared build cache".to_owned(),
11753        holder: busy.describe(),
11754        since: Timestamp::now(),
11755        limit_secs: budget.as_secs(),
11756    });
11757    if let Err(e) = state.save() {
11758        tracing::warn!("could not persist a cache wait: {e:#}");
11759    }
11760    let remaining = budget.saturating_sub(started.elapsed());
11761    let waited = crate::cache::wait_for_unless(
11762        &home,
11763        cache_dir,
11764        owner,
11765        remaining,
11766        Duration::from_secs(5),
11767        cancel,
11768    )
11769    .await;
11770    state.resource_wait = None;
11771    let cause = match &waited {
11772        Ok(Some(_)) => None,
11773        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11774        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11775            Some(crate::run::ResourceBlockCause::Timeout {
11776                waited_secs: started.elapsed().as_secs(),
11777                limit_secs: budget.as_secs(),
11778            })
11779        }
11780        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11781    };
11782    if let Some(cause) = cause {
11783        let detail = match &waited {
11784            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11785            Err(error) => format!("{error:#}"),
11786        };
11787        state.resource_block = Some(crate::run::ResourceBlock {
11788            cause,
11789            node: node.to_owned(),
11790            kind: "shared build cache".into(),
11791            detail,
11792            at: Timestamp::now(),
11793        });
11794        state.event(
11795            "resource",
11796            state.resource_block.as_ref().unwrap().description(),
11797        );
11798    }
11799    if let Err(error) = state.save() {
11800        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11801    }
11802    waited
11803}
11804
11805/// The lease state could not be read at all (unreadable lease file, a
11806/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11807/// known to be contended, so a wave degrades to running without a shared cache
11808/// instead of stopping on a hold that [`crate::triage`] could never release.
11809#[derive(Debug)]
11810struct CacheCheckError(String);
11811
11812impl std::fmt::Display for CacheCheckError {
11813    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11814        f.write_str(&self.0)
11815    }
11816}
11817
11818impl std::error::Error for CacheCheckError {}
11819
11820/// Keep a specific acquisition outcome when the caller adds command evidence.
11821fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11822    state
11823        .resource_block
11824        .get_or_insert_with(|| crate::run::ResourceBlock {
11825            cause: crate::run::ResourceBlockCause::Other,
11826            node: node.to_owned(),
11827            kind: "shared build cache".into(),
11828            detail: format!("{error:#}"),
11829            at: Timestamp::now(),
11830        });
11831    if let Err(error) = state.save() {
11832        tracing::warn!("could not persist cache resource block: {error:#}");
11833    }
11834}
11835
11836/// Run `body` — a verify command batch — while holding the shared build
11837/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11838/// never interleave with another borrower's build against the same
11839/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11840/// timeout, or a human's own `magi review`. See the `cache` module doc for
11841/// why this matters more than Cargo's own per-target locking covers — two
11842/// *different* worktrees building the same package name/version into one
11843/// cache directory is a staleness bug, not a lock contention one.
11844///
11845/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11846/// agent waves do. Repository scheduling also has its own bounded,
11847/// cancellable resource-wait policy. Contention must not consume the
11848/// verification budget: `body` receives the full `budget` after both leases
11849/// are acquired and the cache is fresh. Without a shared cache, only
11850/// repository coordination is acquired before handing over the full budget.
11851///
11852/// A lease that cannot be acquired within the resource wait is reported as a single
11853/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11854/// verification — the same shape a spawn failure already takes in
11855/// [`run_commands`], so a caller need not special-case it.
11856#[allow(clippy::too_many_arguments)]
11857async fn with_cache_lease<'s, F, Fut>(
11858    state: &'s mut RunState,
11859    cache_dir: Option<&Path>,
11860    node: &str,
11861    seat: &str,
11862    worktree: &Path,
11863    head: &str,
11864    budget: Duration,
11865    context: &str,
11866    wait: &crate::repo_lock::WaitPolicy,
11867    body: F,
11868) -> Result<(Vec<CommandOutcome>, bool)>
11869where
11870    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11871    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11872{
11873    // Match editing waves: cache first, repository second. In particular,
11874    // pre_gate must never hold the repository while waiting for a wave's cache.
11875    let started = Instant::now();
11876    let home = crate::run::home();
11877    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11878    let guard = if let Some(cache_dir) = cache_dir {
11879        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11880        match acquire_cache_lease(
11881            state,
11882            cache_dir,
11883            &owner,
11884            resource_limit,
11885            node,
11886            context,
11887            &|| wait.cancelled(),
11888        )
11889        .await
11890        .and_then(|g| {
11891            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11892        }) {
11893            Ok(g) => Some(g),
11894            Err(e) => {
11895                record_cache_error(state, node, &e);
11896                return Ok((
11897                    vec![CommandOutcome {
11898                        command: "(waiting for the shared build cache)".into(),
11899                        code: None,
11900                        output_tail: e.to_string(),
11901                        duration_ms: started.elapsed().as_millis() as u64,
11902                        resource_blocked: true,
11903                    }],
11904                    false,
11905                ));
11906            }
11907        }
11908    } else {
11909        None
11910    };
11911    let _coordination = coordinate_operation(state, worktree, wait).await?;
11912    if let Some(cache_dir) = cache_dir {
11913        let identity = crate::cache::Identity::new(worktree, head);
11914        if let Err(e) = crate::cache::ensure_fresh(
11915            &home,
11916            cache_dir,
11917            &identity,
11918            &state.config.verify.cargo_manifests(),
11919        ) {
11920            record_cache_error(state, node, &e);
11921            state.event(
11922                "verify",
11923                format!(
11924                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11925                    worktree.display(),
11926                    short(head)
11927                ),
11928            );
11929            if let Some(guard) = guard {
11930                guard.release();
11931            }
11932            return Ok((
11933                vec![CommandOutcome {
11934                    command: "(confirming the shared build cache is fresh)".into(),
11935                    code: None,
11936                    output_tail: e.to_string(),
11937                    duration_ms: started.elapsed().as_millis() as u64,
11938                    resource_blocked: true,
11939                }],
11940                false,
11941            ));
11942        }
11943    }
11944    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11945    if !timed_out_pids.is_empty() {
11946        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11947    }
11948    // Drop repository coordination before releasing the cache, reversing the
11949    // acquisition order. A cancelled/error future also drops both guards.
11950    drop(_coordination);
11951    if let Some(guard) = guard {
11952        guard.release();
11953    }
11954    Ok((outcomes, retried))
11955}
11956
11957/// Poll `pids` — commands [`run_commands`] reports as still running when its
11958/// own timeout elapsed — until every one is confirmed gone, or
11959/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11960///
11961/// Real confirmation where confirmation is possible, not a substitute for
11962/// full process-tree observation: a grandchild the timed-out process spawned
11963/// and that survives independently of it is invisible to a pid check the
11964/// same way it always was, and continuing to observe and collect *that*
11965/// stays a different piece of work with its own owner. This only narrows a
11966/// fixed blind wait into an actual check of the pids this process does know
11967/// about.
11968async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11969    wait_for_pids_with(
11970        pids,
11971        crate::proc::pid_alive,
11972        LEASE_RELEASE_POLL,
11973        LEASE_RELEASE_MAX_WAIT,
11974    )
11975    .await;
11976}
11977
11978/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
11979/// interval and ceiling supplied by the caller, so the polling *logic* -
11980/// returns as soon as every pid reports dead, gives up at the ceiling
11981/// otherwise - is testable on millisecond durations without asking the real
11982/// OS about a pid at all.
11983async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11984    pids: &[u32],
11985    alive: F,
11986    poll: Duration,
11987    max_wait: Duration,
11988) {
11989    let deadline = Instant::now() + max_wait;
11990    loop {
11991        if pids.iter().all(|&pid| !alive(pid)) {
11992            return;
11993        }
11994        if Instant::now() >= deadline {
11995            return;
11996        }
11997        tokio::time::sleep(poll).await;
11998    }
11999}
12000
12001/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
12002/// admission that it could not even get a verify command to run, as opposed
12003/// to evidence the command actually produced? A caller that would otherwise
12004/// read a resource-blocked outcome as a red command must check this first:
12005/// see [`Runner::gate`], which retries rather than records `Blocked` when
12006/// this is true.
12007fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
12008    outcomes.iter().any(|o| o.resource_blocked)
12009}
12010
12011/// What [`Runner::gate_fix_round`] decided.
12012pub(crate) enum GateFix {
12013    /// The tree changed and `verify.e2e` is still green: run the gate again.
12014    Retry,
12015    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
12016    /// last failure stands and the run ends blocked.
12017    Stop,
12018    /// `verify.e2e` could not run after the fix (magi's own contention):
12019    /// decide nothing now, a later reentry retries.
12020    Defer,
12021}
12022
12023/// Is every red command in `outcomes` an ordinary failure the code could
12024/// explain: it ran, exited non-zero, and said something?
12025///
12026/// A timeout, a spawn failure and a killed process all leave `code` `None`;
12027/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
12028/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
12029/// what the command is stays the gate's business.
12030pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12031    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12032    red.peek().is_some()
12033        && red.all(|o| {
12034            !o.resource_blocked
12035                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12036                && !o.output_tail.trim().is_empty()
12037        })
12038}
12039
12040/// Describe one verify command's outcome for the event log, distinguishing a
12041/// build/link failure — the toolchain never produced a binary to run — from
12042/// an actual test failure, since only the latter is a verdict on the patch.
12043fn e2e_outcome_label(o: &CommandOutcome) -> String {
12044    if o.ok() {
12045        return "pass".to_owned();
12046    }
12047    let reason = if o.build_failed() {
12048        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12049    } else {
12050        format!("FAIL ({:?})", o.code)
12051    };
12052    format!(
12053        "{reason}\n{}",
12054        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12055    )
12056}
12057
12058/// Run `verify.e2e`, retrying once if the first attempt could not build or
12059/// link — a build/link failure is frequently a race against a shared
12060/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
12061/// `verify` event per command, tagged with `context` (normally `"round N"`)
12062/// so the two call sites that need this — the ordinary per-round leg in
12063/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
12064/// it will ever call a round green — read identically in the event log.
12065async fn run_e2e_with_retry(
12066    state: &mut RunState,
12067    shell: &[String],
12068    commands: &[String],
12069    worktree: &Path,
12070    timeout: Duration,
12071    context: &str,
12072) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12073    let (mut e2e, mut timed_out_pids) = run_commands(
12074        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12075    )
12076    .await;
12077    for o in &e2e {
12078        state.event(
12079            "verify",
12080            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12081        );
12082    }
12083    // A build/link failure is not a verdict on the patch — it is frequently a
12084    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
12085    // one retry before letting a red like that decide the round.
12086    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12087    if verify_retried {
12088        state.event(
12089            "verify",
12090            format!(
12091                "{context}: verify could not build/link, not a test result — retrying once \
12092                 before concluding"
12093            ),
12094        );
12095        let retried = run_commands(
12096            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12097        )
12098        .await;
12099        e2e = retried.0;
12100        // Both attempts' timeouts matter, not just the last one: the first
12101        // attempt's descendants may still be alive alongside the retry's.
12102        timed_out_pids.extend(retried.1);
12103        for o in &e2e {
12104            state.event(
12105                "verify",
12106                format!(
12107                    "{context}: retry `{}` -> {}",
12108                    o.command,
12109                    e2e_outcome_label(o)
12110                ),
12111            );
12112        }
12113    }
12114    (e2e, verify_retried, timed_out_pids)
12115}
12116
12117/// Run configured shell commands in `cwd`, in order. The second element is
12118/// the pid of every command that hit `timeout` and was still running when
12119/// this stopped waiting on it (best-effort: `None` when the platform did not
12120/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
12121/// that releases a shared resource afterward needs to know.
12122///
12123/// Records `task` into [`RunState::active`] at every command boundary
12124/// (`RunState::task_command`) and clears it once the whole list has run
12125/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
12126/// for minutes with no seat and no output of its own to show for it (see
12127/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
12128/// apart from "not yet run" without this), and this is the only place that
12129/// knows which command is running right now and how many are left. Three
12130/// saves per command — start, not per second — matching the same "only at a
12131/// boundary" rule [`wave`] already follows for seats.
12132#[allow(clippy::too_many_arguments)]
12133async fn run_commands(
12134    state: &mut RunState,
12135    node: &str,
12136    task: &str,
12137    attempt: usize,
12138    shell: &[String],
12139    commands: &[String],
12140    cwd: &Path,
12141    timeout: Duration,
12142) -> (Vec<CommandOutcome>, Vec<u32>) {
12143    if commands.is_empty() {
12144        // Nothing to mark as running and nothing to clear — an empty list
12145        // means "not configured", and touching `active` (or the disk) over
12146        // that would be a write for every round of a repo with no
12147        // `verify.e2e` / `verify.gate` commands at all.
12148        return (Vec::new(), Vec::new());
12149    }
12150    let mut out = Vec::new();
12151    let mut timed_out_pids = Vec::new();
12152    let total = commands.len();
12153    for (idx, command) in commands.iter().enumerate() {
12154        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12155        if let Err(e) = state.save() {
12156            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12157        }
12158        let started = Instant::now();
12159        let mut cmd = tokio::process::Command::new(&shell[0]);
12160        cmd.quiet();
12161        // Verify the checkout as a standalone caller. Worker ownership and
12162        // pinned executable selection belong to this driver, not to tests or
12163        // tools it runs. Preserve home/seat context and Running's containment.
12164        for key in crate::worker::INVOCATION_ENV {
12165            cmd.env_remove(key);
12166        }
12167        cmd.args(&shell[1..])
12168            .arg(command)
12169            .current_dir(cwd)
12170            .stdin(std::process::Stdio::null())
12171            .stdout(std::process::Stdio::piped())
12172            .stderr(std::process::Stdio::piped())
12173            .kill_on_drop(true);
12174        let mut outputs = Vec::new();
12175        let protected = (|| -> Result<()> {
12176            crate::incremental::configure(&mut cmd, cwd)?;
12177            let mut paths = vec![cwd.to_owned()];
12178            if let Some(cache) = state.config.cache_dir() {
12179                paths.push(cache);
12180            }
12181            outputs.extend(crate::retention::OutputLease::admit(
12182                &crate::run::home(),
12183                &paths,
12184                state.config.disk.min_free_bytes,
12185            )?);
12186            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12187                outputs.push(guard);
12188            }
12189            if let Some(cache) = state.config.cache_dir() {
12190                outputs.push(crate::retention::OutputLease::cache(
12191                    &crate::run::home(),
12192                    &cache,
12193                    false,
12194                )?);
12195            }
12196            Ok(())
12197        })();
12198        if let Err(e) = protected {
12199            let detail = format!("execution admission unavailable: {e:#}");
12200            state.resource_block = Some(crate::run::ResourceBlock {
12201                cause: crate::run::ResourceBlockCause::Other,
12202                node: node.to_owned(),
12203                kind: if detail.contains("disk admission") {
12204                    "disk admission"
12205                } else {
12206                    "output ownership"
12207                }
12208                .to_owned(),
12209                detail: detail.clone(),
12210                at: Timestamp::now(),
12211            });
12212            out.push(CommandOutcome {
12213                command: command.clone(),
12214                code: None,
12215                output_tail: detail,
12216                duration_ms: 0,
12217                resource_blocked: true,
12218            });
12219            break;
12220        }
12221        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12222        let (code, body) = match spawned {
12223            Ok(mut running) => {
12224                let pid = running.child().id();
12225                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12226                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12227                let mut paths = vec![cwd.to_owned()];
12228                if let Some(cache) = state.config.cache_dir() {
12229                    paths.push(cache);
12230                }
12231                let mut pressure = None;
12232                let status = tokio::select! {
12233                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12234                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12235                        pressure = Some(reason); None
12236                    }
12237                };
12238                let (code, failure) = match status {
12239                    Some(Ok(Ok(status))) => (
12240                        status.code(),
12241                        status
12242                            .code()
12243                            .is_none()
12244                            .then(|| format!("command terminated: {status}")),
12245                    ),
12246                    Some(Ok(Err(error))) => {
12247                        (None, Some(format!("failed waiting for command: {error}")))
12248                    }
12249                    Some(Err(_)) => {
12250                        // Leased cleanup confirms process-tree death itself.
12251                        // Unleased commands still need the caller's PID check
12252                        // after kill-on-drop, before releasing cache ownership.
12253                        if !running.coordinated()
12254                            && let Some(pid) = pid
12255                        {
12256                            timed_out_pids.push(pid);
12257                        }
12258                        (
12259                            None,
12260                            Some(format!("timed out after {}s", timeout.as_secs())),
12261                        )
12262                    }
12263                    None => (None, None),
12264                };
12265                running.finish().await;
12266                if let Some(reason) = pressure {
12267                    state.resource_block = Some(crate::run::ResourceBlock {
12268                        cause: crate::run::ResourceBlockCause::Other,
12269                        node: node.to_owned(),
12270                        kind: "disk pressure".to_owned(),
12271                        detail: reason.clone(),
12272                        at: Timestamp::now(),
12273                    });
12274                    out.push(CommandOutcome {
12275                        command: command.clone(),
12276                        code: None,
12277                        output_tail: reason,
12278                        duration_ms: started.elapsed().as_millis() as u64,
12279                        resource_blocked: true,
12280                    });
12281                    break;
12282                }
12283                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12284                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12285                if let Some(failure) = failure {
12286                    body.push('\n');
12287                    body.push_str(&failure);
12288                }
12289                (code, body)
12290            }
12291            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12292        };
12293        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12294        if !body.is_empty() {
12295            // The full text is kept whatever the summary dropped. A fresh
12296            // name per command so a resume or retry never overwrites an
12297            // earlier run's output. A failed write changes no verdict.
12298            match crate::run::write_verification_artifact(state, task, &body) {
12299                Ok(name) if summary != body => {
12300                    summary = format!("[full output: artifacts/{name}]\n{summary}");
12301                }
12302                Ok(_) => {}
12303                Err(e) => {
12304                    let warning = format!("could not save the full {task} output: {e:#}");
12305                    tracing::warn!("{warning}");
12306                    state.event(node, warning);
12307                    if summary != body {
12308                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12309                    }
12310                }
12311            }
12312        }
12313        let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12314        out.push(CommandOutcome {
12315            command: command.clone(),
12316            code,
12317            output_tail: summary,
12318            duration_ms: started.elapsed().as_millis() as u64,
12319            resource_blocked,
12320        });
12321        if resource_blocked {
12322            state.resource_block = Some(crate::run::ResourceBlock {
12323                cause: crate::run::ResourceBlockCause::Other,
12324                node: node.to_owned(),
12325                kind: "incremental cache admission".into(),
12326                detail: body,
12327                at: Timestamp::now(),
12328            });
12329            break;
12330        }
12331    }
12332    state.task_finished(task);
12333    if let Err(e) = state.save() {
12334        tracing::warn!("could not persist the end of {task}: {e:#}");
12335    }
12336    (out, timed_out_pids)
12337}
12338
12339/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
12340/// section (`report::run`) and in the `merge` event this node records — for
12341/// the operator to run by hand.
12342///
12343/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
12344/// branch whose ruleset forbids merge commits (GitHub's "must not contain
12345/// merge commits", or "require linear history") rejects the push a `--no-ff`
12346/// merge would produce, which is exactly the guidance this function replaces.
12347/// `message`'s first line becomes the squash commit's subject, matching the
12348/// note `report::run` prints alongside this command — see that function for
12349/// why an explicit subject is not optional there.
12350fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12351    let repo = repo.display();
12352    match style {
12353        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12354        MergeStyle::Squash => {
12355            // The subject sits inside double quotes, and a title an agent
12356            // wrote may carry the characters that break out of them.
12357            let subject = message
12358                .lines()
12359                .next()
12360                .unwrap_or(branch)
12361                .replace(['\\', '"', '$', '`'], "");
12362            format!(
12363                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12364            )
12365        }
12366        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12367    }
12368}
12369
12370/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
12371/// under the hood, rejects a `title` over 256 characters and the whole
12372/// command fails — no PR at all, for a run whose body was otherwise fine
12373/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
12374/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
12375/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
12376/// this bound. It is a margin, not a guarantee — a title packed
12377/// with multi-unit characters could still in principle land close to the
12378/// edge, but a real task title's occasional emoji or accented letter fits
12379/// comfortably inside it.
12380const PR_TITLE_MAX: usize = 240;
12381
12382/// A pull request title from the opening line of `text`, or `None` when that
12383/// line is not English (GitHub text is) or has no letters.
12384///
12385/// A line within `max` is kept as is. A longer one is cut at the end of its
12386/// first sentence when that falls inside `max`, else at a word boundary with a
12387/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
12388/// make every merely-truncated title look non-English). The language check
12389/// runs on the kept text before any mark is added, so only what GitHub will
12390/// show is judged: an English opening followed by non-ASCII far past the cut
12391/// still passes.
12392fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12393    let line = queue::first_line(text)?;
12394    let chars: Vec<char> = line.chars().collect();
12395    let (kept, mark) = if chars.len() <= max {
12396        (line.to_owned(), "")
12397    } else if let Some(end) = sentence_end(&chars, max) {
12398        (chars[..end].iter().collect::<String>(), "")
12399    } else {
12400        let room = max.saturating_sub(3);
12401        // Cut at the last space inside the room; when the char just past the
12402        // room is a space the room already ends on a word.
12403        let cut = if chars[room].is_whitespace() {
12404            room
12405        } else {
12406            chars[..room]
12407                .iter()
12408                .rposition(|c| c.is_whitespace())
12409                .unwrap_or(room)
12410        };
12411        let head: String = chars[..cut].iter().collect();
12412        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12413        (head.to_owned(), "...")
12414    };
12415    if kept.is_empty() {
12416        return None;
12417    }
12418    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12419        return None;
12420    }
12421    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12422        return None;
12423    }
12424    Some(format!("{kept}{mark}"))
12425}
12426
12427/// The char length of the first sentence of `chars` when it ends within `max`
12428/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
12429/// abbreviations so `e.g. foo` does not end a title early.
12430fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12431    const MIN: usize = 20;
12432    for i in MIN..max.min(chars.len()) {
12433        if !matches!(chars[i], '.' | '!' | '?') {
12434            continue;
12435        }
12436        let Some(&next) = chars.get(i + 1) else {
12437            continue;
12438        };
12439        if !next.is_whitespace() {
12440            continue;
12441        }
12442        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12443        if after.is_some_and(|c| c.is_ascii_lowercase()) {
12444            continue;
12445        }
12446        let word: String = chars[..i]
12447            .iter()
12448            .rev()
12449            .take_while(|c| !c.is_whitespace())
12450            .collect::<Vec<_>>()
12451            .into_iter()
12452            .rev()
12453            .collect();
12454        let word = word.to_ascii_lowercase();
12455        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12456            continue;
12457        }
12458        let end = chars[..i]
12459            .iter()
12460            .rposition(|c| !c.is_whitespace())
12461            .map_or(i, |p| p + 1);
12462        return Some(end);
12463    }
12464    None
12465}
12466
12467/// What `merge = "pr"` (and the merge commit of the other modes) says about a
12468/// change: a title and a body describing what was *implemented*, not the task
12469/// that asked for it. A task reads as a request; a reader of the merged
12470/// history wants the change.
12471struct PrMessage {
12472    title: String,
12473    body: String,
12474}
12475
12476impl PrMessage {
12477    /// Title, blank line, body. The first line is the squash/merge commit
12478    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
12479    /// has to stay one sensible line.
12480    fn commit_message(&self) -> String {
12481        format!("{}\n\n{}", self.title, self.body)
12482    }
12483}
12484
12485/// The text after a leading `TITLE:` (any case) on `line`.
12486fn title_marker(line: &str) -> Option<&str> {
12487    let line = line.trim();
12488    let head = line.get(..6)?;
12489    head.eq_ignore_ascii_case("title:")
12490        .then(|| line[6..].trim())
12491}
12492
12493/// The implementer's own one-line title: the `TITLE:` line the implement
12494/// prompt asks for at the top of its SUMMARY. Candidate commits are all
12495/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
12496/// source, and a title that says as much is refused here too.
12497fn summary_title(summary: &str) -> Option<String> {
12498    let first = summary.lines().find(|l| !l.trim().is_empty())?;
12499    let raw = title_marker(first)?;
12500    if raw.is_empty() {
12501        return None;
12502    }
12503    let title = queue::title_from(raw, PR_TITLE_MAX);
12504    let lower = title.to_ascii_lowercase();
12505    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12506        return None;
12507    }
12508    Some(title)
12509}
12510
12511/// How `open_review`'s instruction begins; see [`landing_title`].
12512const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12513
12514/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12515pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12516
12517/// Does this run review work that already existed, rather than implement a
12518/// task? Runs recorded before `reviewed_commits` existed carry only the
12519/// candidate marker.
12520pub(crate) fn is_review_run(state: &RunState) -> bool {
12521    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12522}
12523
12524/// The title of a review-only run: the subject of the oldest commit under
12525/// review. Later commits are usually fixups, and `instruction` is the review
12526/// prompt, which says nothing about the change. GitHub text is English, so a
12527/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12528fn review_title(state: &RunState) -> Option<String> {
12529    english_subject(
12530        state.reviewed_commits.as_ref()?.first()?,
12531        state.config.graph.github_english(),
12532    )
12533}
12534
12535/// `raw` as a pull request title, or `None` when it is blank, not English
12536/// (GitHub text is), or one of magi's own candidate commit subjects.
12537fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12538    let raw = raw.trim();
12539    if raw.is_empty() {
12540        return None;
12541    }
12542    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12543        return None;
12544    }
12545    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12546        return None;
12547    }
12548    let title = queue::title_from(raw, PR_TITLE_MAX);
12549    let lower = title.to_ascii_lowercase();
12550    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12551        return None;
12552    }
12553    Some(title)
12554}
12555
12556/// What a review-only run's branch says about itself, read at the moment the
12557/// pull request is opened.
12558#[derive(Debug, Clone, PartialEq, Eq)]
12559struct BranchFacts {
12560    /// `(subject, body)` of each commit, oldest first.
12561    commits: Vec<(String, String)>,
12562    /// Trimmed `git diff --stat`.
12563    stat: String,
12564}
12565
12566/// Longest diff stat shown: this many file lines plus the summary line.
12567const STAT_FILE_LINES: usize = 25;
12568/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12569const COMMITS_MAX_CHARS: usize = 20_000;
12570
12571/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12572/// say or finds nothing, so the caller falls back to what the run recorded.
12573async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12574    let commits = git::commit_log(repo, base, branch).await.ok()?;
12575    if commits.is_empty() {
12576        return None;
12577    }
12578    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12579    let lines: Vec<&str> = stat.lines().collect();
12580    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12581        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12582        let more = format!(" ... {omitted} more file(s)");
12583        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12584        kept.push(&more);
12585        kept.push(lines[lines.len() - 1]);
12586        kept.join("\n")
12587    } else {
12588        lines.join("\n")
12589    };
12590    Some(BranchFacts { commits, stat })
12591}
12592
12593/// Defang what would break the surrounding markdown: a closing `</details>`
12594/// and a code fence.
12595fn markdown_safe(text: &str) -> String {
12596    text.replace("</details>", "&lt;/details&gt;")
12597        .replace("\x60\x60\x60", "~~~")
12598}
12599
12600fn neutral_title(state: &RunState, winner: char) -> String {
12601    format!(
12602        "chore: land candidate {} of run {}",
12603        winner.to_ascii_uppercase(),
12604        state.id
12605    )
12606}
12607
12608/// Reuse the graph's bounded fixer without starting another graph or driver.
12609/// Copy the working state back even on errors so spent rounds remain visible.
12610pub(crate) async fn repair_landing_gate(
12611    state: &mut RunState,
12612    winner: &Candidate,
12613    outcomes: &[CommandOutcome],
12614) -> Result<GateFix> {
12615    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12616        return Ok(GateFix::Stop);
12617    }
12618    let mut runner = Runner {
12619        roles: state.config.resolve_roles()?,
12620        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12621        state: state.clone(),
12622        pause: Pause::new(),
12623        interrupt: Pause::new(),
12624    };
12625    let result = runner.gate_fix_round(winner, outcomes).await;
12626    *state = runner.state;
12627    result
12628}
12629
12630/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12631/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12632pub(crate) async fn verify_current_head(
12633    state: &mut RunState,
12634    winner: &Candidate,
12635    wait: &crate::repo_lock::WaitPolicy,
12636) -> Result<bool> {
12637    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12638    // Preserve the previous record before replacing it, including on crash recovery.
12639    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12640        state
12641            .verification_history
12642            .push(crate::run::VerificationEvidence {
12643                superseded_by: Some(head.clone()),
12644                archived_at: Some(Timestamp::now()),
12645                fixes_spent: Some(state.gate_fixes.len()),
12646                gate_head: None,
12647                gate_ran: false,
12648                gate: Vec::new(),
12649                e2e: state.head_e2e.take(),
12650            });
12651    }
12652    if let Some(e) = &state.head_e2e
12653        && e.head == head
12654    {
12655        if e.passed {
12656            return Ok(true);
12657        }
12658        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12659            state.status = RunStatus::Blocked;
12660            state.save()?;
12661            return Ok(false);
12662        }
12663    } else if state.reviews.last().is_some_and(|r| {
12664        r.verified_head.as_deref() == Some(head.as_str())
12665            && (r.e2e_status() == E2eStatus::Passed
12666                || (r.e2e_status() == E2eStatus::NotConfigured
12667                    && state.config.verify.e2e.is_empty()))
12668    }) || state
12669        .gate_repair_e2e
12670        .as_ref()
12671        .is_some_and(|e| e.head == head && e.passed)
12672    {
12673        return Ok(true);
12674    }
12675    if let Some(e) = &state.gate_repair_e2e
12676        && e.head == head
12677        && !e.passed
12678        && !e.outcomes.is_empty()
12679        && !verify_inconclusive(&e.outcomes)
12680    {
12681        state.status = RunStatus::Blocked;
12682        state.save()?;
12683        return Ok(false);
12684    }
12685    state.head_e2e = Some(crate::run::GateRepairE2e {
12686        head: head.clone(),
12687        passed: false,
12688        outcomes: Vec::new(),
12689    });
12690    state.save()?;
12691    let commands = state.config.verify.e2e.clone();
12692    let shell = state.config.shell();
12693    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12694    let cache_dir = state.config.cache_dir();
12695    let outcomes = if commands.is_empty() {
12696        Vec::new()
12697    } else {
12698        let (outcomes, _) = with_cache_lease(
12699            state,
12700            cache_dir.as_deref(),
12701            "e2e",
12702            "e2e",
12703            &winner.worktree,
12704            &head,
12705            timeout,
12706            "current head e2e",
12707            wait,
12708            |state, budget| async move {
12709                run_e2e_with_retry(
12710                    state,
12711                    &shell,
12712                    &commands,
12713                    &winner.worktree,
12714                    budget,
12715                    "current head e2e",
12716                )
12717                .await
12718            },
12719        )
12720        .await?;
12721        outcomes
12722    };
12723    let unresolved = verify_inconclusive(&outcomes);
12724    let passed = !unresolved
12725        && outcomes.iter().all(CommandOutcome::ok)
12726        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12727    if !outcomes.is_empty() && !unresolved {
12728        state.record_base_progress(&head);
12729    }
12730    state.head_e2e = Some(crate::run::GateRepairE2e {
12731        head: head.clone(),
12732        passed,
12733        outcomes,
12734    });
12735    if !passed && !unresolved {
12736        state.status = RunStatus::Blocked;
12737        state.event(
12738            "e2e",
12739            format!("verify.e2e failed on head {head}; nothing published"),
12740        );
12741    }
12742    state.save()?;
12743    Ok(passed)
12744}
12745
12746/// Recheck a repair's e2e requirement across failures, resource deferrals and
12747/// process interruptions. A successful result only applies to its exact SHA.
12748pub(crate) async fn check_gate_repair_e2e(
12749    state: &mut RunState,
12750    winner: &Candidate,
12751    wait: &crate::repo_lock::WaitPolicy,
12752) -> Result<GateFix> {
12753    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12754        return Ok(GateFix::Retry);
12755    }
12756    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12757    if state
12758        .gate_repair_e2e
12759        .as_ref()
12760        .is_some_and(|r| r.head == head && r.passed)
12761    {
12762        return Ok(GateFix::Retry);
12763    }
12764    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12765        head: head.clone(),
12766        passed: false,
12767        outcomes: Vec::new(),
12768    });
12769    state.save()?;
12770    let commands = state.config.verify.e2e.clone();
12771    let outcomes = if commands.is_empty() {
12772        Vec::new()
12773    } else {
12774        let shell = state.config.shell();
12775        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12776        let cache_dir = state.config.cache_dir();
12777        let (outcomes, _) = with_cache_lease(
12778            state,
12779            cache_dir.as_deref(),
12780            "e2e",
12781            "e2e",
12782            &winner.worktree,
12783            &head,
12784            timeout,
12785            "gate repair e2e",
12786            wait,
12787            |state, budget| async move {
12788                run_e2e_with_retry(
12789                    state,
12790                    &shell,
12791                    &commands,
12792                    &winner.worktree,
12793                    budget,
12794                    "gate repair e2e",
12795                )
12796                .await
12797            },
12798        )
12799        .await?;
12800        outcomes
12801    };
12802    let deferred = verify_inconclusive(&outcomes);
12803    let passed = outcomes.iter().all(CommandOutcome::ok)
12804        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12805    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12806        head,
12807        passed,
12808        outcomes,
12809    });
12810    state.save()?;
12811    if deferred {
12812        Ok(GateFix::Defer)
12813    } else if !passed {
12814        state.event(
12815            "gate",
12816            "verify.e2e failed after gate repair; nothing published",
12817        );
12818        Ok(GateFix::Stop)
12819    } else {
12820        Ok(GateFix::Retry)
12821    }
12822}
12823
12824/// Run the same configured gate for a landing head and a fresh graph.
12825pub(crate) async fn run_landing_gate(
12826    state: &mut RunState,
12827    winner: &Candidate,
12828    wait: &crate::repo_lock::WaitPolicy,
12829) -> Result<Vec<CommandOutcome>> {
12830    let shell = state.config.shell();
12831    let gate_commands = state.config.verify.gate.clone();
12832    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12833    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12834        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12835        Vec::new()
12836    } else {
12837        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12838        let cache_dir = state.config.cache_dir();
12839        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12840        let (outcomes, _) = with_cache_lease(
12841            state,
12842            cache_dir.as_deref(),
12843            "gate",
12844            "gate",
12845            &winner.worktree,
12846            &head,
12847            timeout,
12848            "final gate",
12849            wait,
12850            |state, budget| {
12851                let shell = shell.clone();
12852                let gate_commands = gate_commands.clone();
12853                let winner = winner.clone();
12854                async move {
12855                    Runner::run_pre_gate(state, &winner).await;
12856                    // Bind command evidence after mechanical fixes, before commands start.
12857                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12858                    let (outcomes, timed_out_pids) = run_commands(
12859                        state,
12860                        "gate",
12861                        "gate",
12862                        0,
12863                        &shell,
12864                        &gate_commands,
12865                        &winner.worktree,
12866                        budget,
12867                    )
12868                    .await;
12869                    (outcomes, false, timed_out_pids)
12870                }
12871            },
12872        )
12873        .await?;
12874        outcomes
12875    };
12876    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12877    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12878        let mut outcomes = outcomes;
12879        outcomes.push(CommandOutcome {
12880            command: "gate head stability".into(), code: None,
12881            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12882            duration_ms: 0, resource_blocked: false,
12883        });
12884        return Ok(outcomes);
12885    }
12886    if outcomes.is_empty() {
12887        // Nothing configured to check — distinct from every other
12888        // silence in this run's event log, since an empty `gate` alone
12889        // no longer says whether the gate ran at all (see
12890        // `RunState::gate_ran`'s own doc).
12891        state.event("gate", "no gate commands configured; not tested");
12892    }
12893    for o in &outcomes {
12894        state.event(
12895            "gate",
12896            format!(
12897                "`{}` -> {}",
12898                o.command,
12899                if o.ok() {
12900                    "pass".to_owned()
12901                } else {
12902                    format!(
12903                        "FAIL ({:?})\n{}",
12904                        o.code,
12905                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12906                    )
12907                }
12908            ),
12909        );
12910    }
12911    Ok(outcomes)
12912}
12913
12914/// The pull request title to hand to `land::merge_subject`. A review-only run
12915/// opened by an earlier build titled its pull request with the review prompt;
12916/// that title is dropped (empty, so the fallback applies) rather than landed.
12917/// Any other title, including an operator's rename, passes through untouched,
12918/// and so does every title of a run that implements a task.
12919pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12920    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12921        ""
12922    } else {
12923        pr_title
12924    }
12925}
12926
12927/// What the squash subject falls back to when the pull request title is empty
12928/// or candidate-shaped: for a review-only run the derived title, never the
12929/// review prompt held in `instruction`.
12930pub fn landing_subject_source(state: &RunState) -> String {
12931    if is_review_run(state) {
12932        let winner = state.candidates.first().map_or('A', |c| c.label);
12933        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12934    }
12935    state.instruction.clone()
12936}
12937
12938/// `summary` without its `TITLE:` line, which the pull request title already
12939/// carries.
12940fn summary_without_title(summary: &str) -> String {
12941    let mut lines = summary.trim().lines().peekable();
12942    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12943        lines.next();
12944    }
12945    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12946}
12947
12948/// The pull request title and body for the winning candidate.
12949///
12950/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12951/// the task's own opening line via [`queue::title_from`] when there is none.
12952/// `state.instruction` can open with blank lines (`task_text` only rejects a
12953/// body that is blank *entirely*), which `title_from` skips.
12954///
12955/// Publishing uses an independent description generated from the final diff
12956/// and structured evidence. Internal summaries and reports remain local.
12957/// Non-publishing previews may display the original task and candidate summary.
12958/// The footer retains the run and candidate provenance.
12959#[cfg(test)]
12960fn pr_message(state: &RunState, winner: char) -> PrMessage {
12961    pr_message_with(state, winner, None)
12962}
12963
12964/// [`pr_message`] with what the branch of a review-only run says about itself.
12965/// `facts` is ignored for a run that implements a task.
12966#[cfg(test)]
12967fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12968    let raw = pr_message_raw(
12969        state,
12970        winner,
12971        facts,
12972        crate::pr_outcome::FinalHead::Unreported,
12973        ChangeNote::NotAsked,
12974    );
12975    let id = crate::scrub::Identity::current();
12976    PrMessage {
12977        title: crate::scrub::scrub(&raw.title, &id),
12978        body: crate::scrub::scrub(&raw.body, &id),
12979    }
12980}
12981
12982/// What the public description may say about the change itself.
12983#[derive(Debug, Clone, Copy)]
12984enum ChangeNote<'a> {
12985    /// A local preview or non-publishing merge mode: no public generation needed.
12986    NotAsked,
12987    /// A summary written from the final diff.
12988    Summary(&'a crate::run::ChangeSummary),
12989    /// A summary was asked for and could not be produced; the text says why.
12990    Unavailable(&'a str),
12991}
12992
12993fn pr_message_raw(
12994    state: &RunState,
12995    winner: char,
12996    facts: Option<&BranchFacts>,
12997    head: crate::pr_outcome::FinalHead<'_>,
12998    change: ChangeNote<'_>,
12999) -> PrMessage {
13000    if !matches!(change, ChangeNote::NotAsked) {
13001        let (title, mut body) = match change {
13002            ChangeNote::Summary(c) => (
13003                c.title.clone(),
13004                format!("## Summary\n\n{}\n\n", c.body.trim()),
13005            ),
13006            ChangeNote::Unavailable(why) => (
13007                neutral_title(state, winner),
13008                format!(
13009                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
13010                ),
13011            ),
13012            ChangeNote::NotAsked => unreachable!(),
13013        };
13014        body.push_str(&public_pr_evidence(state, head));
13015        body.push_str(&format!(
13016            "\n---\nmagi:run/{} magi:candidate-{}\n",
13017            state.id,
13018            winner.to_ascii_lowercase()
13019        ));
13020        return PrMessage { title, body };
13021    }
13022    pr_preview(state, winner, facts, head)
13023}
13024
13025/// Append public facts from the run records after every prose generation,
13026/// including posting-gate rewrites. The agent cannot drop this provenance.
13027fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13028    let mut body = crate::pr_outcome::review_rounds(state);
13029    body.push_str(&crate::pr_outcome::final_verification(state, head));
13030    let open = state.last_round_findings();
13031    if !open.is_empty() {
13032        body.push_str("## Open review findings\n\n");
13033        for f in open {
13034            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13035        }
13036        body.push('\n');
13037    }
13038    if let Some(c) = state.change_summary.as_ref()
13039        && head == crate::pr_outcome::FinalHead::At(&c.head)
13040    {
13041        body.push_str(&format!(
13042            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13043            c.base, c.head
13044        ));
13045    }
13046    body
13047}
13048
13049/// Local-only preview of the operator's task and candidate summary.
13050fn pr_preview(
13051    state: &RunState,
13052    winner: char,
13053    facts: Option<&BranchFacts>,
13054    head: crate::pr_outcome::FinalHead<'_>,
13055) -> PrMessage {
13056    let summary = state
13057        .candidates
13058        .iter()
13059        .find(|c| c.label == winner)
13060        .map(|c| c.summary.as_str())
13061        .unwrap_or_default();
13062    // The fallback is the operator's own words and may not be English; GitHub
13063    // text always is, so a non-English task gets a neutral title instead.
13064    let review = is_review_run(state);
13065    let title = if review {
13066        facts
13067            .and_then(|f| {
13068                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13069            })
13070            .or_else(|| review_title(state))
13071            .or_else(|| {
13072                state
13073                    .candidates
13074                    .iter()
13075                    .find(|c| c.label == winner)
13076                    .filter(|c| !c.branch.starts_with("magi/"))
13077                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13078            })
13079            .unwrap_or_else(|| neutral_title(state, winner))
13080    } else {
13081        summary_title(summary).unwrap_or_else(|| {
13082            english_title(
13083                &state.instruction,
13084                PR_TITLE_MAX,
13085                state.config.graph.github_english(),
13086            )
13087            .unwrap_or_else(|| neutral_title(state, winner))
13088        })
13089    };
13090
13091    let mut body = String::new();
13092    let what = summary_without_title(summary);
13093    if !what.is_empty() {
13094        body.push_str("## Summary\n\n");
13095        body.push_str(&what);
13096        body.push_str("\n\n");
13097    }
13098
13099    // Review history uses structured facts across all rounds, never reports.
13100    body.push_str(&crate::pr_outcome::review_rounds(state));
13101    body.push_str(&crate::pr_outcome::final_verification(state, head));
13102    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13103
13104    let open = state.open_findings();
13105    if !open.is_empty() {
13106        body.push_str("## Open review findings\n\n");
13107        for f in &open {
13108            body.push_str(&format!(
13109                "- `{}` [{:?}] {}\n",
13110                f.id, f.severity, "Context retained in local review records."
13111            ));
13112        }
13113        body.push('\n');
13114    }
13115
13116    if let Some(fix) = fix
13117        && !fix.rejected.is_empty()
13118    {
13119        body.push_str("## Declined by the fixer\n\n");
13120        for r in &fix.rejected {
13121            body.push_str(&format!(
13122                "- `{}`: {}\n",
13123                r.id, "Reason retained in local review records."
13124            ));
13125        }
13126        body.push('\n');
13127    }
13128
13129    if review {
13130        // The review prompt is not the task; list what the branch carries.
13131        body.push_str("## Commits under review\n\n");
13132        if let Some(facts) = facts {
13133            let mut left = COMMITS_MAX_CHARS;
13134            for (i, (subject, text)) in facts.commits.iter().enumerate() {
13135                let mut entry = format!("- {}\n", markdown_safe(subject));
13136                for l in markdown_safe(text).lines() {
13137                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
13138                }
13139                if left == 0 {
13140                    body.push_str(&format!(
13141                        "- ... {} more commit(s)\n",
13142                        facts.commits.len() - i
13143                    ));
13144                    break;
13145                }
13146                if entry.len() > left {
13147                    // Even the first commit is cut: one huge body must not
13148                    // push the whole description past GitHub's limit.
13149                    let mut end = left;
13150                    while !entry.is_char_boundary(end) {
13151                        end -= 1;
13152                    }
13153                    entry.truncate(end);
13154                    entry.push_str("\n  ... (truncated)\n");
13155                    left = 0;
13156                } else {
13157                    left -= entry.len();
13158                }
13159                body.push_str(&entry);
13160            }
13161            if !facts.stat.trim().is_empty() {
13162                body.push_str(&format!(
13163                    "\n## Diff stat\n\n```\n{}\n```\n",
13164                    markdown_safe(facts.stat.trim())
13165                ));
13166            }
13167        } else {
13168            match &state.reviewed_commits {
13169                Some(subjects) => {
13170                    for s in subjects {
13171                        body.push_str(&format!("- {}\n", s.trim()));
13172                    }
13173                }
13174                None => {
13175                    // An older run kept only the prompt, with the commit list
13176                    // after its first paragraph.
13177                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13178                    body.push_str(rest.trim());
13179                    body.push('\n');
13180                }
13181            }
13182        }
13183    } else {
13184        let task = state.instruction.trim();
13185        let task = if task.is_empty() {
13186            "(empty task)"
13187        } else {
13188            task
13189        };
13190        body.push_str(&format!(
13191            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13192            task.replace("</details>", "&lt;/details&gt;")
13193        ));
13194    }
13195
13196    body.push_str(&format!(
13197        "\n---\nmagi:run/{} magi:candidate-{}\n",
13198        state.id,
13199        winner.to_ascii_lowercase()
13200    ));
13201
13202    PrMessage { title, body }
13203}
13204
13205/// Source imports have already been prepared from structured inputs. Legacy
13206/// inferred references must never instruct a seat to reconstruct those imports.
13207fn seeded_instruction(state: &RunState) -> String {
13208    let mut inputs = Vec::new();
13209    if let Some(start) = &state.sources.start_from {
13210        inputs.push(format!(
13211            "Start from `{}` at `{}` (including ancestry).",
13212            start.token, start.sha
13213        ));
13214    }
13215    for pick in &state.sources.pick_commits {
13216        inputs.push(format!(
13217            "Pick `{}` at `{}` (single commit).",
13218            pick.token, pick.sha
13219        ));
13220    }
13221    if inputs.is_empty() {
13222        state.instruction.clone()
13223    } else {
13224        format!(
13225            "{}\n\n# Explicit source inputs already prepared\n\n{}",
13226            state.instruction,
13227            inputs.join("\n")
13228        )
13229    }
13230}
13231
13232/// Does the winner have no commits ahead of the base it would land on?
13233/// Any failure to find out reads as "not empty": the merge then behaves as it
13234/// always did rather than refusing on a guess.
13235async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13236    let base = &state.base_branch;
13237    let mut against = base.clone();
13238    if mode == MergeMode::Pr {
13239        // A pull request lands on the remote's base, never the local branch
13240        // of the same name: if that cannot be read, "not empty" is the safe
13241        // answer.
13242        let remote = &state.config.merge.remote;
13243        let tracking = format!("{remote}/{base}");
13244        let fetched = git::fetch(repo, remote, base).await;
13245        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13246            against = tracking;
13247        } else {
13248            return false;
13249        }
13250    }
13251    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13252}
13253
13254/// Why nothing was opened for an empty winner, with what the task's own
13255/// references resolved to.
13256fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13257    let mut detail = format!(
13258        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13259         nothing to open a pull request for"
13260    );
13261    detail.push_str(
13262        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13263    );
13264    detail
13265}
13266
13267/// What the `Pr` merge does once it knows whether the branch already has an
13268/// open pull request.
13269#[derive(Debug, PartialEq, Eq)]
13270enum PrPlan {
13271    Create,
13272    Adopt { url: String, title: String },
13273    Stop(String),
13274}
13275
13276/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
13277/// a failed lookup -> stop with the real reason. Never guesses.
13278fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13279    match found {
13280        Ok(land::OpenPr::None) => PrPlan::Create,
13281        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13282        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13283            "several open pull requests exist for this branch, not picking one: {}",
13284            urls.join(" ")
13285        )),
13286        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13287    }
13288}
13289
13290/// `gh pr create`, returning the PR url.
13291pub(crate) async fn gh_pr_create(
13292    cwd: &Path,
13293    base: &str,
13294    head: &str,
13295    title: &str,
13296    body: &str,
13297) -> Result<String> {
13298    let out = tokio::process::Command::new("gh")
13299        .args([
13300            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13301        ])
13302        .current_dir(cwd)
13303        .quiet()
13304        .stdin(std::process::Stdio::null())
13305        .output()
13306        .await
13307        .context("spawn gh")?;
13308    if out.status.success() {
13309        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13310    } else {
13311        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13312    }
13313}
13314
13315/// Never force deletion of uncommitted work, including unknown legacy trees.
13316async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13317    match std::fs::symlink_metadata(path) {
13318        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13319            // A previous cleanup may have removed the tree before a branch
13320            // deletion or state save failed. Unregister it if still present.
13321            git::worktree_remove_clean(repo, path).await?;
13322            return Ok(());
13323        }
13324        Err(e) => return Err(e.into()),
13325        Ok(_) => {}
13326    }
13327    if !git::is_clean(path).await? {
13328        bail!(
13329            "worktree has uncommitted work; refusing removal: {}",
13330            path.display()
13331        );
13332    }
13333    if !git::worktree_remove_clean(repo, path).await? {
13334        bail!("worktree removal refused: {}", path.display());
13335    }
13336    Ok(())
13337}
13338
13339/// Read seats return replies, not working-tree changes. Only registered,
13340/// detached seats directly inside this run's root may discard those changes.
13341async fn remove_read_seat_worktree(state: &mut RunState, path: &Path) -> Result<()> {
13342    if !crate::retention::read_seat(path) || !path.exists() {
13343        return remove_preserved_worktree(&state.repo, path).await;
13344    }
13345    let owned = std::fs::symlink_metadata(path)?.is_dir()
13346        && path.parent().map(Path::canonicalize).transpose()?
13347            == Some(state.worktree_root().canonicalize()?)
13348        && crate::retention::worktrees(state)
13349            .await?
13350            .contains(&path.canonicalize()?);
13351    let common = crate::repo_lock::common_dir(&state.repo).await?;
13352    if !owned
13353        || common.is_none()
13354        || crate::repo_lock::common_dir(path).await? != common
13355        || git::git_raw(path, &["symbolic-ref", "-q", "HEAD"])
13356            .await?
13357            .ok()
13358        || state.candidates.iter().any(|c| c.worktree == path)
13359    {
13360        return remove_preserved_worktree(&state.repo, path).await;
13361    }
13362    let changes = git::status_porcelain(path).await?;
13363    if !git::worktree_remove(&state.repo, path).await? {
13364        bail!(
13365            "read-only seat worktree removal refused: {}",
13366            path.display()
13367        );
13368    }
13369    if !changes.is_empty() {
13370        let paths = changes.lines().take(20).collect::<Vec<_>>().join("; ");
13371        let paths: String = paths.chars().take(1024).collect();
13372        state.event(
13373            "fold",
13374            format!(
13375                "discarded read-only seat {} changes ({} status entries; first 20, at most 1024 chars): {paths}",
13376                path.file_name().unwrap_or_default().to_string_lossy(),
13377                changes.lines().count()
13378            ),
13379        );
13380    }
13381    Ok(())
13382}
13383
13384/// Tear a run's worktrees and branches down.
13385///
13386/// `home` is where the updated `run.json` is saved (via
13387/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
13388/// a housekeeping pass already has its own honest `home` handed to it, and
13389/// falling through to the global here would write back through whichever
13390/// directory some other process or test pinned into that `OnceLock` first,
13391/// not the one the caller actually resolved its `runs` and `state` from.
13392pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13393    fold_run_with_policy(state, drop_winner, home, false).await
13394}
13395
13396/// Explicit CLI-only retention override; automatic callers use [`fold_run`].
13397pub async fn fold_run_overriding_recovery_retention(
13398    state: &mut RunState,
13399    drop_winner: bool,
13400    home: &Path,
13401) -> Result<Vec<String>> {
13402    fold_run_with_policy(state, drop_winner, home, true).await
13403}
13404
13405async fn fold_run_with_policy(
13406    state: &mut RunState,
13407    drop_winner: bool,
13408    home: &Path,
13409    override_recovery_retention: bool,
13410) -> Result<Vec<String>> {
13411    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state)
13412        && !override_recovery_retention
13413    {
13414        bail!("refusing fold: {reason}");
13415    }
13416    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13417        bail!("candidate preservation incomplete; refusing cleanup");
13418    }
13419    // Serialize ordinary folding with the same starts/resumes as target-only
13420    // eviction. Otherwise a second janitor could remove the whole worktree
13421    // around a live output borrower even though target eviction was fenced.
13422    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13423    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13424    // An override must use current durable ownership, not a pre-lock snapshot.
13425    if override_recovery_retention {
13426        *state = RunState::load_under(&state.id, home)?;
13427        anyhow::ensure!(
13428            state.status.done() && !state.status.resumable(),
13429            "recovery retention override requires a finished, non-resumable run"
13430        );
13431        let claimed = crate::daemon::is_working_on_other(home, &state.id, Timestamp::now())?;
13432        anyhow::ensure!(
13433            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
13434            "run is live or its ownership is unknown"
13435        );
13436        anyhow::ensure!(
13437            state.candidate_preservation.is_empty() && state.interrupted_implementations.is_empty(),
13438            "candidate preservation incomplete; refusing cleanup"
13439        );
13440    }
13441    let retention_reason = crate::pr_recovery::source_retention_reason(home, state);
13442    if let Some(reason) = &retention_reason
13443        && !override_recovery_retention
13444    {
13445        bail!("refusing fold: {reason}");
13446    }
13447    let _driver = crate::driver::maintenance(home, &state.id)?;
13448    let mut output_guards = Vec::new();
13449    for entry in std::fs::read_dir(state.worktree_root())
13450        .into_iter()
13451        .flatten()
13452        .flatten()
13453    {
13454        if entry.path().join(".git").is_file() {
13455            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13456        }
13457    }
13458    let repo = state.repo.clone();
13459    // CLI, Web and automatic cleanup all enter here, independently of
13460    // Runner::fold_losers. Protect deletion and shared config release too.
13461    let wait = repository_wait(state, None);
13462    let _coordination = match coordinate_operation(state, &repo, &wait).await {
13463        Ok(lease) => lease,
13464        Err(e) => {
13465            state.save_under(home)?;
13466            return Err(e);
13467        }
13468    };
13469    // Confirmed merges dispose of their owned sources after durable evidence
13470    // capture. Preflight every source before removing anything.
13471    let _source_outputs = if retention_reason.is_some()
13472        || (state.status == RunStatus::Merged && state.merge.as_ref().is_some_and(|merge| merge.ok))
13473    {
13474        verify_merged_fold_sources(state, drop_winner, home).await?
13475    } else {
13476        Vec::new()
13477    };
13478    if let Some(reason) = retention_reason {
13479        state.event(
13480            "fold",
13481            format!(
13482                "operator CLI --override-recovery-retention authorized deletion for run {}: bypassed {reason}",
13483                state.id
13484            ),
13485        );
13486        state.save_under(home)?;
13487    }
13488    let root = state.worktree_root();
13489    let winner = state.tally.as_ref().map(|t| t.winner);
13490    let mut removed = Vec::new();
13491
13492    for i in 0..state.candidates.len() {
13493        let c = state.candidates[i].clone();
13494        let is_winner = Some(c.label) == winner;
13495        if is_winner && !drop_winner {
13496            continue;
13497        }
13498        let existed = c.worktree.exists();
13499        // A vanished directory can still have a Git registration that keeps
13500        // its branch checked out. Remove that registration before the branch.
13501        remove_preserved_worktree(&repo, &c.worktree).await?;
13502        if existed {
13503            removed.push(c.worktree.to_string_lossy().into_owned());
13504        }
13505        // A branch handed to a later run (and its pull request) is not this
13506        // run's to delete.
13507        let handed_over = state.released_branches.contains(&c.branch);
13508        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13509            if !git::branch_delete(&repo, &c.branch).await? {
13510                bail!("candidate branch removal refused");
13511            }
13512            removed.push(c.branch.clone());
13513        }
13514        state.candidates[i].folded = true;
13515    }
13516
13517    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13518        let path = name.path();
13519        let keep = !drop_winner
13520            && winner.is_some_and(|w| {
13521                path.file_name()
13522                    .is_some_and(|n| n == format!("cand-{w}").as_str())
13523            });
13524        if keep {
13525            continue;
13526        }
13527        remove_read_seat_worktree(state, &path).await?;
13528        removed.push(path.to_string_lossy().into_owned());
13529    }
13530
13531    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
13532    // judge worktrees, so once the loop above has cleared all of them out,
13533    // the parent is a bare directory nobody else was ever going to remove -
13534    // git only ever managed what was inside it. Left alone, one of these
13535    // accumulates per fully-folded run; the operator's own machine had 74.
13536    // `remove_if_empty` re-checks rather than assuming: a run whose winner
13537    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
13538    // so does anything a run never claimed that happens to share the bay.
13539    remove_if_empty(&root);
13540
13541    if state.enabled_worktree_config && drop_winner {
13542        // A release, not a raw disable: some sibling run in this repository
13543        // may still hold its own reference (see `git::acquire_worktree_config`),
13544        // and only the last release actually turns the setting back off.
13545        git::release_worktree_config(&repo).await.ok();
13546        state.enabled_worktree_config = false;
13547    }
13548    state.save_under(home)?;
13549    Ok(removed)
13550}
13551
13552async fn verify_merged_fold_sources(
13553    state: &RunState,
13554    drop_winner: bool,
13555    home: &Path,
13556) -> Result<Vec<std::fs::File>> {
13557    let winner = state.tally.as_ref().map(|t| t.winner);
13558    let mut heads = BTreeSet::new();
13559    for candidate in &state.candidates {
13560        if Some(candidate.label) == winner && !drop_winner {
13561            continue;
13562        }
13563        anyhow::ensure!(
13564            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13565            "candidate recovery source is outside the inventoried run root: {}",
13566            candidate.worktree.display()
13567        );
13568        match std::fs::symlink_metadata(&candidate.worktree) {
13569            Ok(_) => {}
13570            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13571            Err(error) => return Err(error.into()),
13572        }
13573        if !state.released_branches.contains(&candidate.branch)
13574            && git::branch_exists(&state.repo, &candidate.branch).await?
13575        {
13576            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13577        }
13578    }
13579    let mut cargo_guards = Vec::new();
13580    let entries = match std::fs::read_dir(state.worktree_root()) {
13581        Ok(entries) => Some(entries),
13582        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13583        Err(error) => return Err(error.into()),
13584    };
13585    for entry in entries.into_iter().flatten() {
13586        let path = entry?.path();
13587        let metadata = std::fs::symlink_metadata(&path)?;
13588        anyhow::ensure!(
13589            metadata.is_dir() && !metadata.file_type().is_symlink(),
13590            "non-directory or symlink recovery source retained: {}",
13591            path.display()
13592        );
13593        if !drop_winner
13594            && winner.is_some_and(|w| {
13595                path.file_name()
13596                    .is_some_and(|name| name == format!("cand-{w}").as_str())
13597            })
13598        {
13599            continue;
13600        }
13601        anyhow::ensure!(
13602            path.join(".git").is_file(),
13603            "unregistered recovery source retained: {}",
13604            path.display()
13605        );
13606        anyhow::ensure!(
13607            git::git(
13608                &path,
13609                &[
13610                    "status",
13611                    "--porcelain",
13612                    "--untracked-files=all",
13613                    "--ignored",
13614                    "--",
13615                    ".",
13616                    ":(exclude)target"
13617                ]
13618            )
13619            .await?
13620            .is_empty(),
13621            "source changes or ignored recovery files retained: {}",
13622            path.display()
13623        );
13624        match std::fs::symlink_metadata(path.join("target")) {
13625            Ok(_) => {
13626                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13627            }
13628            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13629            Err(error) => return Err(error.into()),
13630        }
13631        for checkpoint in [
13632            "rebase-merge",
13633            "rebase-apply",
13634            "sequencer",
13635            "CHERRY_PICK_HEAD",
13636            "MERGE_HEAD",
13637            "REBASE_HEAD",
13638            "REVERT_HEAD",
13639            "BISECT_LOG",
13640            "AUTO_MERGE",
13641            "MERGE_MSG",
13642        ] {
13643            let location = git::git(
13644                &path,
13645                &[
13646                    "rev-parse",
13647                    "--path-format=absolute",
13648                    "--git-path",
13649                    checkpoint,
13650                ],
13651            )
13652            .await?;
13653            anyhow::ensure!(
13654                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13655                if e.kind() == std::io::ErrorKind::NotFound),
13656                "Git recovery checkpoint retained: {checkpoint}"
13657            );
13658        }
13659        heads.insert(git::rev_parse(&path, "HEAD").await?);
13660    }
13661    crate::fold_evidence::preserve(state, home, &heads).await?;
13662
13663    Ok(cargo_guards)
13664}
13665
13666/// Remove `dir` if it exists and has nothing in it.
13667///
13668/// Best-effort and silent by design: a directory that is not empty (a run
13669/// whose winner is still parked there, a stray file some other process left)
13670/// is exactly the case this must refuse, and a directory that is already gone
13671/// is not a failure worth reporting either. `std::fs::remove_dir` itself
13672/// already refuses a non-empty directory, so the emptiness check below is
13673/// belt, not suspenders - it is what keeps this from ever attempting the
13674/// removal in the case that matters, rather than trusting `remove_dir`'s
13675/// error path to have no side effects if it ever changed.
13676fn remove_if_empty(dir: &Path) {
13677    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13678        std::fs::remove_dir(dir).ok();
13679    }
13680}
13681
13682/// Severity of the worst open finding in the last review round, for reporting.
13683pub fn worst_open(state: &RunState) -> Option<Severity> {
13684    state
13685        .reviews
13686        .last()?
13687        .reviews
13688        .iter()
13689        .flat_map(|r| r.findings.iter())
13690        .map(|f| f.severity)
13691        .max()
13692}
13693
13694/// `[worktree] setup` hides its products per worktree, which needs
13695/// `extensions.worktreeConfig`. Taken once per run and released with the run's
13696/// worktrees, like the hook's; every place that runs setup calls this first.
13697async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13698    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13699        git::acquire_worktree_config(repo).await?;
13700        state.enabled_worktree_config = true;
13701    }
13702    Ok(())
13703}
13704
13705#[cfg(test)]
13706mod tests {
13707    #[tokio::test]
13708    async fn launcher_base_fetch_requires_remote_success() {
13709        let tmp = tempfile::tempdir().unwrap();
13710        let repo = tmp.path().join("repo");
13711        let remote = tmp.path().join("remote");
13712        std::fs::create_dir(&repo).unwrap();
13713        std::fs::create_dir(&remote).unwrap();
13714        git::git(&remote, &["init", "--bare", "-b", "main"])
13715            .await
13716            .unwrap();
13717        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13718        git::git(
13719            &repo,
13720            &[
13721                "-c",
13722                "user.name=Fixture",
13723                "-c",
13724                "user.email=fixture@example.invalid",
13725                "commit",
13726                "--allow-empty",
13727                "-m",
13728                "fixture",
13729            ],
13730        )
13731        .await
13732        .unwrap();
13733        git::git(
13734            &repo,
13735            &["remote", "add", "origin", remote.to_str().unwrap()],
13736        )
13737        .await
13738        .unwrap();
13739        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13740        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13741        assert_eq!(
13742            resolve_base(&repo, "main", "origin", None)
13743                .await
13744                .unwrap()
13745                .commit,
13746            head
13747        );
13748        #[cfg(unix)]
13749        {
13750            let policy: crate::config::Coordination = toml::from_str(
13751                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13752            ).unwrap();
13753            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13754            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13755            let held = crate::repo_lock::SeatLease::operation_with(
13756                &repo,
13757                Some(&policy),
13758                &crate::repo_lock::WaitPolicy::default(),
13759            )
13760            .await
13761            .unwrap();
13762            assert!(held.is_some());
13763            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13764            tokio::pin!(fetch);
13765            assert!(
13766                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13767                    .await
13768                    .is_err()
13769            );
13770            assert!(
13771                !repo.join(".git/FETCH_HEAD").exists(),
13772                "no fetch before admission"
13773            );
13774            drop(held);
13775            let fetched = fetch.await.unwrap();
13776            assert_eq!(fetched.commit, head);
13777            assert_eq!(fetched.remote, "origin");
13778            assert_eq!(fetched.branch, "main");
13779        }
13780        // A surviving tracking ref is not successful fetch evidence.
13781        git::git(
13782            &repo,
13783            &[
13784                "remote",
13785                "set-url",
13786                "origin",
13787                tmp.path().join("missing").to_str().unwrap(),
13788            ],
13789        )
13790        .await
13791        .unwrap();
13792        let error = resolve_base(&repo, "main", "origin", None)
13793            .await
13794            .unwrap_err()
13795            .to_string();
13796        assert!(error.contains("cannot read origin/main"));
13797        assert!(error.contains("refusing to branch off"));
13798    }
13799
13800    #[test]
13801    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13802        let leaked = vec!["chore(deps): update a crate".to_owned()];
13803        let own = "fix(daemon): apply a chosen action";
13804        assert!(should_retitle("", own, &leaked));
13805        assert!(should_retitle(
13806            &format!("{REVIEW_PROMPT_OPENING} `x`"),
13807            own,
13808            &leaked
13809        ));
13810        assert!(should_retitle(
13811            "chore: land candidate A of run 1",
13812            own,
13813            &leaked
13814        ));
13815        assert!(should_retitle(
13816            "magi: candidate A (uncommitted work)",
13817            own,
13818            &leaked
13819        ));
13820        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13821        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13822        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13823    }
13824
13825    #[test]
13826    fn pr_merge_plan_creates_adopts_or_stops() {
13827        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13828        assert_eq!(
13829            pr_merge_plan(Ok(land::OpenPr::One {
13830                url: "u".into(),
13831                title: "t".into()
13832            })),
13833            PrPlan::Adopt {
13834                url: "u".into(),
13835                title: "t".into()
13836            }
13837        );
13838        let PrPlan::Stop(many) =
13839            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13840        else {
13841            panic!("many must stop");
13842        };
13843        assert!(many.contains('a') && many.contains('b'));
13844        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13845            panic!("a failed lookup must stop");
13846        };
13847        assert!(err.contains("bad token"));
13848    }
13849
13850    use super::*;
13851    use crate::run::GateStatus;
13852    use std::collections::BTreeMap;
13853    use std::time::Duration;
13854
13855    fn conductor() -> AgentSpec {
13856        AgentSpec {
13857            id: "conductor".to_owned(),
13858            kind: crate::config::AgentKind::Command,
13859            model: None,
13860            command: vec!["true".to_owned()],
13861            extra_args: Vec::new(),
13862            env: BTreeMap::new(),
13863            prompt_delivery: None,
13864            max_turn_secs: None,
13865        }
13866    }
13867
13868    fn spec(id: &str) -> AgentSpec {
13869        AgentSpec {
13870            id: id.to_owned(),
13871            kind: crate::config::AgentKind::Command,
13872            model: None,
13873            command: vec!["true".to_owned()],
13874            extra_args: Vec::new(),
13875            env: BTreeMap::new(),
13876            prompt_delivery: None,
13877            max_turn_secs: None,
13878        }
13879    }
13880
13881    fn ids(xs: &[&str]) -> BTreeSet<String> {
13882        xs.iter().map(|s| (*s).to_owned()).collect()
13883    }
13884
13885    #[test]
13886    fn pick_successor_skips_an_agent_another_seat_holds() {
13887        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
13888        let roster = [spec("a"), spec("b"), spec("c")];
13889        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13890        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13891    }
13892
13893    #[test]
13894    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13895        // The other seat started on c but was handed to d; c is free again.
13896        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13897        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13898        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13899        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13900        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13901    }
13902
13903    #[test]
13904    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13905        let roster = [spec("a"), spec("b")];
13906        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13907        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13908        let carried = ids(&["b"]);
13909        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13910        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13911    }
13912
13913    #[test]
13914    fn pick_successor_returns_none_without_any_untried_successor() {
13915        let roster = [spec("a"), spec("b")];
13916        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13917        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13918        let carried = ids(&["a"]);
13919        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13920    }
13921
13922    #[test]
13923    fn pick_successor_rescue_avoids_another_seats_occupant() {
13924        // b is a carried failure and free; a is held by the other seat.
13925        let roster = [spec("a"), spec("b"), spec("c")];
13926        let carried = ids(&["b", "c"]);
13927        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13928        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13929    }
13930
13931    #[test]
13932    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13933        let roster = [spec("a"), spec("b"), spec("c")];
13934        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13935        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13936    }
13937
13938    #[test]
13939    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13940        let roster = [spec("a"), spec("b"), spec("c")];
13941        let failed = ids(&["a", "b", "c"]);
13942        // Rescue looks at the whole roster, once per id, then runs out.
13943        let mut tried = ids(&["b"]);
13944        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13945        assert_eq!(first.id, "a");
13946        tried.insert(first.id.clone());
13947        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13948        assert_eq!(second.id, "c");
13949        tried.insert(second.id.clone());
13950        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13951    }
13952
13953    #[test]
13954    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13955        let roster = [spec("a"), spec("b")];
13956        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13957        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13958    }
13959
13960    #[test]
13961    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13962        let roster = [spec("a"), spec("b"), spec("c")];
13963        let h = SeatHistory {
13964            failed: ids(&["a"]),
13965            last_ok: Some("b".to_owned()),
13966            last_fail: None,
13967        };
13968        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13969        // A last answerer that left the roster, or later failed, is ignored.
13970        let gone = SeatHistory {
13971            last_ok: Some("zzz".to_owned()),
13972            ..h.clone()
13973        };
13974        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13975        let failed = SeatHistory {
13976            failed: ids(&["a", "b"]),
13977            last_ok: Some("b".to_owned()),
13978            last_fail: None,
13979        };
13980        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13981    }
13982
13983    #[test]
13984    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13985        let first = SeatState::new("review-1", "alpha", 7);
13986        let next = handover_seat("review-1", "gamma", 7);
13987        assert_ne!(first.claude_session, next.claude_session);
13988    }
13989
13990    #[test]
13991    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13992        let mut state = state_with_summary("x", "y");
13993        let a = handover_seat("review-1", "beta", state.next_seat_seed());
13994        let b = handover_seat("review-1", "beta", state.next_seat_seed());
13995        assert_ne!(a.claude_session, b.claude_session);
13996    }
13997
13998    #[test]
13999    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
14000        let roster = [spec("a"), spec("b")];
14001        let h = SeatHistory {
14002            failed: ids(&["a", "b"]),
14003            ..SeatHistory::default()
14004        };
14005        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
14006        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
14007        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
14008    }
14009
14010    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
14011    // fallback loop depends on to terminate: it must walk forward from the
14012    // seat's own position, never restart at the front of the roster, and it
14013    // must never hand back an id already tried, however many times that id
14014    // happens to appear.
14015
14016    #[test]
14017    fn failure_signature_ignores_numbers_and_paths() {
14018        assert_eq!(
14019            failure_signature("exited with Some(2) and no usable output"),
14020            failure_signature("exited with Some(137) and no usable output")
14021        );
14022        assert_eq!(
14023            failure_signature("process exited with no usable output (exit code 2)"),
14024            failure_signature("process exited with no usable output (exit code 137)")
14025        );
14026        for (old, new) in [
14027            (
14028                "exited with Some(2) and no usable output",
14029                "process exited with no usable output (exit code 2)",
14030            ),
14031            (
14032                "exited with None and no usable output",
14033                "process exited with no usable output (exit code unavailable)",
14034            ),
14035        ] {
14036            let previous = FailClass::Other(failure_signature(old));
14037            let current = FailClass::Other(failure_signature(new));
14038            assert_eq!(previous, current, "resumption retains the failure class");
14039            assert!(!should_hand_over(Some(&previous), &current));
14040        }
14041        assert_eq!(
14042            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
14043            failure_signature("cannot open /var/x.txt: denied")
14044        );
14045        assert_ne!(failure_signature("boom"), failure_signature("bang"));
14046    }
14047
14048    #[test]
14049    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
14050        let other = FailClass::Other("x".into());
14051        assert!(should_hand_over(None, &FailClass::Quota));
14052        assert!(should_hand_over(Some(&other), &FailClass::Quota));
14053        assert!(should_hand_over(
14054            Some(&FailClass::Timeout),
14055            &FailClass::Timeout
14056        ));
14057        assert!(should_hand_over(None, &other));
14058        assert!(!should_hand_over(Some(&other), &other));
14059        assert!(should_hand_over(
14060            Some(&other),
14061            &FailClass::Other("y".into())
14062        ));
14063        // A quota or timeout in between ends the run of identical failures.
14064        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
14065        assert!(should_hand_over(Some(&FailClass::Quota), &other));
14066    }
14067
14068    #[test]
14069    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
14070        let a = SeatState::new("judge-1", "alpha", 7);
14071        let b = handover_seat("judge-1", "beta", 7);
14072        assert_ne!(a.claude_session, b.claude_session);
14073        assert_eq!(b.turns, 0);
14074    }
14075
14076    #[test]
14077    fn a_timeout_is_classified_apart_from_other_failures() {
14078        assert_eq!(
14079            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
14080            Some(FailClass::Timeout)
14081        );
14082        assert!(matches!(
14083            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
14084            Some(FailClass::Other(_))
14085        ));
14086    }
14087
14088    #[test]
14089    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
14090        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14091        let tried = BTreeSet::from(["beta".to_owned()]);
14092        // beta sits at index 1; the next candidate is gamma, never alpha —
14093        // which is very likely a different candidate slot's own agent.
14094        let next = next_untried_in_roster(&roster, 1, &tried);
14095        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
14096    }
14097
14098    #[test]
14099    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
14100        let roster = vec![spec("alpha"), spec("beta")];
14101        let tried = BTreeSet::from(["beta".to_owned()]);
14102        // beta is the roster's last entry: nothing follows it, and alpha —
14103        // earlier in the roster, almost certainly a different candidate
14104        // slot's own agent — must not be reached by wrapping back to it.
14105        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14106    }
14107
14108    #[test]
14109    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
14110        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14111        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
14112        // beta (index 1) and gamma (index 2, the only entry after it) have
14113        // both been tried; alpha (index 0) never has, but it comes before
14114        // beta's own position, so there is nothing further for this seat.
14115        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14116    }
14117
14118    #[test]
14119    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14120        let roster = vec![spec("a"), spec("a"), spec("b")];
14121        let tried = BTreeSet::from(["a".to_owned()]);
14122        let next = next_untried_in_roster(&roster, 0, &tried);
14123        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14124    }
14125
14126    #[test]
14127    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14128        let roster = vec![spec("a"), spec("b")];
14129        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14130        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14131    }
14132
14133    #[test]
14134    fn remove_if_empty_only_ever_takes_a_bare_directory() {
14135        let dir = tempfile::tempdir().unwrap();
14136        let bay = dir.path().join("ffff");
14137
14138        // Not there yet: nothing to do, nothing to panic on.
14139        remove_if_empty(&bay);
14140        assert!(!bay.exists());
14141
14142        // Something still inside - the winner's worktree, or a stray file -
14143        // keeps the directory standing.
14144        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14145        remove_if_empty(&bay);
14146        assert!(bay.exists(), "non-empty directory must survive");
14147
14148        // Once the last entry is gone, so is the directory itself.
14149        std::fs::remove_dir(bay.join("cand-A")).unwrap();
14150        remove_if_empty(&bay);
14151        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14152    }
14153
14154    // `round_is_clean` is the exact decision this task fixed: a round with a
14155    // seat that never answered must not read the same as a round every seat
14156    // actually reviewed. These are deterministic and process-free by design —
14157    // the equivalent end-to-end check (a real reviewer timing out under a
14158    // live graph run) is a genuine race against wall-clock contention, and a
14159    // spawn slow enough to blow even a generous budget under a loaded test
14160    // run must not turn this specific regression check flaky.
14161
14162    #[test]
14163    fn a_full_panel_that_found_nothing_is_clean() {
14164        assert!(round_is_clean(
14165            0,
14166            true,
14167            2,
14168            2,
14169            0,
14170            IncompleteReviewPolicy::Block
14171        ));
14172    }
14173
14174    #[test]
14175    fn a_missing_seat_is_never_clean_under_the_default_policy() {
14176        assert!(!round_is_clean(
14177            0,
14178            true,
14179            1,
14180            2,
14181            0,
14182            IncompleteReviewPolicy::Block
14183        ));
14184    }
14185
14186    #[test]
14187    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14188        assert!(!round_is_clean(
14189            1,
14190            true,
14191            1,
14192            2,
14193            0,
14194            IncompleteReviewPolicy::Warn
14195        ));
14196    }
14197
14198    #[test]
14199    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14200        assert!(round_is_clean(
14201            0,
14202            true,
14203            1,
14204            2,
14205            0,
14206            IncompleteReviewPolicy::Warn
14207        ));
14208    }
14209
14210    #[test]
14211    fn a_full_panel_with_an_open_finding_is_not_clean() {
14212        assert!(!round_is_clean(
14213            1,
14214            true,
14215            2,
14216            2,
14217            0,
14218            IncompleteReviewPolicy::Block
14219        ));
14220    }
14221
14222    #[test]
14223    fn a_full_panel_with_a_red_e2e_is_not_clean() {
14224        assert!(!round_is_clean(
14225            0,
14226            false,
14227            2,
14228            2,
14229            0,
14230            IncompleteReviewPolicy::Block
14231        ));
14232    }
14233
14234    // The stall this task closes: under the default `block` policy, a seat
14235    // missing only because it was rate limited must not force a wait for a
14236    // session limit that will not lift by the next round. `round_is_clean`
14237    // is where that quorum carve-out lives; the review loop around it never
14238    // changes what a reviewer's vote or a finding's severity means.
14239
14240    #[test]
14241    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14242        // 1 of 2 answered, and the one missing was quota'd — the exact
14243        // "review-2 rate limited (quota)" shape from the field report.
14244        assert!(round_is_clean(
14245            0,
14246            true,
14247            1,
14248            2,
14249            1,
14250            IncompleteReviewPolicy::Block
14251        ));
14252    }
14253
14254    #[test]
14255    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14256        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
14257        // not a quota loss (`quota_missing` stays 0) — worth another try.
14258        assert!(!round_is_clean(
14259            0,
14260            true,
14261            1,
14262            2,
14263            0,
14264            IncompleteReviewPolicy::Block
14265        ));
14266    }
14267
14268    #[test]
14269    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14270        assert!(!round_is_clean(
14271            1,
14272            true,
14273            1,
14274            2,
14275            1,
14276            IncompleteReviewPolicy::Block
14277        ));
14278        assert!(!round_is_clean(
14279            0,
14280            false,
14281            1,
14282            2,
14283            1,
14284            IncompleteReviewPolicy::Block
14285        ));
14286    }
14287
14288    #[test]
14289    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14290        // Every seat quota'd, nobody answered: there is no panel to decide
14291        // on, so this must fall through to the existing block-and-retry
14292        // fallback rather than call an unreviewed patch clean.
14293        assert!(!round_is_clean(
14294            0,
14295            true,
14296            0,
14297            2,
14298            2,
14299            IncompleteReviewPolicy::Block
14300        ));
14301    }
14302
14303    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14304        CommandOutcome {
14305            command: "test".to_owned(),
14306            code,
14307            output_tail: String::new(),
14308            duration_ms: 0,
14309            resource_blocked,
14310        }
14311    }
14312
14313    #[cfg(unix)]
14314    #[tokio::test]
14315    async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14316        ask_test_home();
14317        let temp = tempfile::tempdir().unwrap();
14318        let mut runner = runner_at(RunStatus::Gating);
14319        let shell = vec!["sh".to_owned(), "-c".to_owned()];
14320        let commands = vec![
14321            "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14322            "touch must-not-run".to_owned(),
14323        ];
14324        let (outcomes, retried, pids) = run_e2e_with_retry(
14325            &mut runner.state,
14326            &shell,
14327            &commands,
14328            temp.path(),
14329            Duration::from_secs(5),
14330            "fixture",
14331        )
14332        .await;
14333        assert_eq!(outcomes.len(), 1);
14334        assert!(outcomes[0].resource_blocked);
14335        assert!(!outcomes[0].build_failed());
14336        assert!(!retried);
14337        assert!(pids.is_empty());
14338        assert!(!temp.path().join("must-not-run").exists());
14339        assert_eq!(
14340            runner.state.resource_block.as_ref().unwrap().kind,
14341            "incremental cache admission"
14342        );
14343    }
14344
14345    #[test]
14346    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14347        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14348        assert!(
14349            !verify_inconclusive(&[outcome(Some(1), false)]),
14350            "an ordinary failure is still evidence about the patch"
14351        );
14352        assert!(verify_inconclusive(&[outcome(None, true)]));
14353        assert!(
14354            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14355            "one inconclusive outcome taints the whole batch"
14356        );
14357        assert!(!verify_inconclusive(&[]));
14358    }
14359
14360    #[tokio::test]
14361    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14362        // Alive for the first two checks, then dead - confirms the loop
14363        // actually re-polls rather than deciding once and sleeping out the
14364        // ceiling regardless.
14365        let calls = std::sync::atomic::AtomicUsize::new(0);
14366        let started = Instant::now();
14367        wait_for_pids_with(
14368            &[123],
14369            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14370            Duration::from_millis(5),
14371            Duration::from_secs(5),
14372        )
14373        .await;
14374        assert!(
14375            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14376            "must keep checking rather than deciding on the first answer"
14377        );
14378        assert!(
14379            started.elapsed() < Duration::from_secs(1),
14380            "must return the moment it is confirmed dead, not wait out the ceiling"
14381        );
14382    }
14383
14384    #[tokio::test]
14385    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14386        let started = Instant::now();
14387        wait_for_pids_with(
14388            &[123],
14389            |_| true, // never reports dead
14390            Duration::from_millis(5),
14391            Duration::from_millis(30),
14392        )
14393        .await;
14394        let elapsed = started.elapsed();
14395        assert!(
14396            elapsed >= Duration::from_millis(30),
14397            "must not give up before its own ceiling: {elapsed:?}"
14398        );
14399        assert!(
14400            elapsed < Duration::from_secs(1),
14401            "must not wait past its own ceiling either: {elapsed:?}"
14402        );
14403    }
14404
14405    #[tokio::test]
14406    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14407        let started = Instant::now();
14408        wait_for_pids_with(
14409            &[],
14410            |_| true,
14411            Duration::from_secs(5),
14412            Duration::from_secs(5),
14413        )
14414        .await;
14415        assert!(
14416            started.elapsed() < Duration::from_millis(200),
14417            "an empty pid list has nothing to confirm"
14418        );
14419    }
14420
14421    // `review_conclusion` is the exact decision the review hand-off task
14422    // fixed: a round budget spent (or a tree that stopped moving) must not
14423    // collapse into `Blocked` regardless of what verification actually
14424    // said. Deterministic and process-free for the same reason the
14425    // `round_is_clean` family above is.
14426    fn review_round(
14427        clean: bool,
14428        blocking: usize,
14429        answered: usize,
14430        expected: usize,
14431        progressed: bool,
14432        e2e_ok: bool,
14433    ) -> ReviewRound {
14434        ReviewRound {
14435            round: 1,
14436            head: "h".to_owned(),
14437            verified_head: None,
14438            verified_at: None,
14439            reviews: Vec::new(),
14440            e2e: vec![CommandOutcome {
14441                command: "test".to_owned(),
14442                code: Some(if e2e_ok { 0 } else { 1 }),
14443                output_tail: String::new(),
14444                duration_ms: 0,
14445                resource_blocked: false,
14446            }],
14447            verify_retried: false,
14448            e2e_deferred: false,
14449            e2e_defer_reason: None,
14450            fix: None,
14451            blocking,
14452            answered,
14453            expected,
14454            clean,
14455            progressed,
14456            vote_split: false,
14457            reconsideration: Vec::new(),
14458            verdict: None,
14459        }
14460    }
14461
14462    #[test]
14463    fn review_conclusion_is_none_when_nothing_has_run() {
14464        assert_eq!(review_conclusion(&[], 3), None);
14465    }
14466
14467    #[test]
14468    fn review_conclusion_is_none_while_rounds_remain() {
14469        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14470        assert_eq!(review_conclusion(&rounds, 3), None);
14471    }
14472
14473    #[test]
14474    fn review_conclusion_is_gating_once_a_round_is_clean() {
14475        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14476        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14477    }
14478
14479    #[test]
14480    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14481        let rounds = vec![
14482            review_round(false, 1, 2, 2, true, true),
14483            review_round(false, 1, 2, 2, true, true),
14484        ];
14485        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14486    }
14487
14488    #[test]
14489    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14490        let rounds = vec![
14491            review_round(false, 1, 2, 2, true, true),
14492            review_round(false, 1, 2, 2, true, false),
14493        ];
14494        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14495    }
14496
14497    #[test]
14498    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14499        // Magi never got a command to run against this round's own head — a
14500        // resource-blocked attempt, not a red one — so this must never
14501        // settle on `Blocked` the way a genuine e2e failure would. `None`
14502        // here is what tells `Runner::review_loop` to retry the check
14503        // itself rather than trust this cheap recomputation with a verdict
14504        // it cannot actually produce.
14505        let mut blocked = review_round(false, 1, 2, 2, true, false);
14506        blocked.e2e[0].resource_blocked = true;
14507        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14508        assert_eq!(review_conclusion(&rounds, 2), None);
14509    }
14510
14511    #[test]
14512    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14513        // Missing input, not a verified tree — never a hand-off candidate.
14514        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14515        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14516    }
14517
14518    #[test]
14519    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14520        let rounds = vec![
14521            review_round(false, 1, 2, 2, false, true),
14522            review_round(false, 1, 2, 2, false, true),
14523        ];
14524        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14525    }
14526
14527    fn secs(n: u64) -> Duration {
14528        Duration::from_secs(n)
14529    }
14530
14531    /// A throwaway repo with one commit on `main`, for tests that need `merge`
14532    /// to make real (and, if it runs at all, real*ly fail*) git calls.
14533    #[tokio::test]
14534    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14535        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14536        if std::env::var_os(CHILD).is_none() {
14537            // Other tests reconcile every run in the process-global home,
14538            // briefly taking owner locks. This ancestry test expects an idle
14539            // predecessor, so give it its own process and pinned test home.
14540            let output = std::process::Command::new(std::env::current_exe().unwrap())
14541                .args([
14542                    "--exact",
14543                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14544                    "--nocapture",
14545                ])
14546                .env(CHILD, "1")
14547                .env_remove("MAGI_WORKER_HOME")
14548                .env_remove("MAGI_WORKER_ID")
14549                .output()
14550                .unwrap();
14551            assert!(
14552                output.status.success()
14553                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14554                "{}\n{}",
14555                String::from_utf8_lossy(&output.stdout),
14556                String::from_utf8_lossy(&output.stderr)
14557            );
14558            return;
14559        }
14560        ask_test_home();
14561        let tmp = tempfile::tempdir().unwrap();
14562        let repo = tmp.path().join("repo");
14563        std::fs::create_dir_all(&repo).unwrap();
14564        init_repo(&repo);
14565        let g = |args: &[&str]| {
14566            let out = std::process::Command::new("git")
14567                .args(args)
14568                .current_dir(&repo)
14569                .quiet()
14570                .output()
14571                .unwrap();
14572            assert!(
14573                out.status.success(),
14574                "{}",
14575                String::from_utf8_lossy(&out.stderr)
14576            );
14577        };
14578        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14579        g(&["checkout", "-b", "historical"]);
14580        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14581        g(&["add", "."]);
14582        g(&["commit", "-m", "old alternative"]);
14583        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14584        g(&["checkout", "-b", "magi/task-selected", &base]);
14585        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14586        g(&["add", "."]);
14587        g(&["commit", "-m", "measurement infrastructure"]);
14588        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14589        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14590        g(&["add", "."]);
14591        g(&["commit", "-m", "dependent implementation"]);
14592        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14593        g(&["checkout", "main"]);
14594        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14595        g(&["add", "."]);
14596        g(&["commit", "-m", "unrelated base change"]);
14597        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14598        let origin = tmp.path().join("origin.git");
14599        g(&[
14600            "clone",
14601            "--bare",
14602            repo.to_str().unwrap(),
14603            origin.to_str().unwrap(),
14604        ]);
14605        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14606        g(&["fetch", "origin"]);
14607        let selection = crate::pr_recovery::Selection {
14608            url: "https://example.test/project/pull/1".into(),
14609            branch: "magi/task-selected".into(),
14610            head: head.clone(),
14611            source_head: None,
14612            source_run: None,
14613            source_proof: None,
14614            disposition: Some(crate::pr_recovery::Disposition::Review),
14615            instruction: "preserve measured behavior".into(),
14616            approved_at: jiff::Timestamp::now(),
14617        };
14618        let mut template = runner_at(RunStatus::Reviewing);
14619        template.state.repo = repo.clone();
14620        template.state.base_commit = current.clone();
14621        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14622        template.state.origin = Some(crate::run::Origin::queue("selected"));
14623        template.state.pr_head = Some(selection.branch.clone());
14624        template.state.pr_recovery = Some(selection.clone());
14625        let mut runner =
14626            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14627                .await
14628                .unwrap();
14629        let wt = runner.state.winner().unwrap().worktree.clone();
14630        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14631        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14632        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14633        assert!(runner.state.seeds.is_empty());
14634        assert_eq!(runner.state.instruction, "preserve measured behavior");
14635        runner.sync_to_base().await.unwrap();
14636        assert_eq!(
14637            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14638            "prerequisite"
14639        );
14640        assert_eq!(
14641            std::fs::read_to_string(wt.join("implementation")).unwrap(),
14642            "measured implementation"
14643        );
14644        assert_eq!(
14645            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14646            "current main change"
14647        );
14648        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14649        assert_eq!(
14650            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14651            head
14652        );
14653
14654        // A failed selected-source setup stays inspectable, never resumable as a new competition.
14655        let mut partial = runner_at(RunStatus::Reviewing);
14656        partial.state.repo = repo.clone();
14657        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14658        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14659            run: Some("exit 1".into()),
14660            ..Default::default()
14661        }];
14662        let selection = runner.state.pr_recovery.clone().unwrap();
14663        partial.state.pr_recovery = Some(selection.clone());
14664        partial.state.pr_head = Some(selection.branch.clone());
14665        partial.state.origin = Some(crate::run::Origin::queue("selected"));
14666        let id = partial.state.id.clone();
14667        let error = match Runner::open_selected(
14668            &repo,
14669            partial.state,
14670            partial.roles,
14671            1,
14672            runner.state.base_commit.clone(),
14673            selection,
14674        )
14675        .await
14676        {
14677            Ok(_) => panic!("setup must fail"),
14678            Err(e) => e,
14679        };
14680        assert_eq!(
14681            error
14682                .downcast_ref::<crate::run::StartupFailure>()
14683                .unwrap()
14684                .run,
14685            id
14686        );
14687        let failed = RunState::load(&id).unwrap();
14688        assert_eq!(failed.status, RunStatus::Failed);
14689        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14690        assert!(Runner::resume(&id).is_err());
14691        runner.state.status = RunStatus::Ready;
14692        runner.state.driver_exited = true;
14693        runner.state.save().unwrap();
14694        let mut retry = runner_at(RunStatus::Reviewing);
14695        retry.state.repo = repo.clone();
14696        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14697        let mut reselected = runner.state.pr_recovery.clone().unwrap();
14698        reselected.approved_at = jiff::Timestamp::now();
14699        retry.state.origin = Some(crate::run::Origin::queue("selected"));
14700        retry.state.pr_head = Some(reselected.branch.clone());
14701        retry.state.pr_recovery = Some(reselected.clone());
14702        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14703            .await
14704            .unwrap();
14705        let restarted = Runner::open_selected(
14706            &repo,
14707            retry.state,
14708            retry.roles,
14709            1,
14710            runner.state.base_commit.clone(),
14711            reselected,
14712        )
14713        .await
14714        .unwrap();
14715        assert_eq!(
14716            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14717                .await
14718                .unwrap(),
14719            head
14720        );
14721        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14722        assert_eq!(
14723            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14724            head
14725        );
14726    }
14727
14728    #[tokio::test]
14729    #[cfg(unix)]
14730    async fn fallback_and_interrupted_restoration_require_coordination() {
14731        use std::os::fd::AsRawFd as _;
14732        ask_test_home();
14733        let tmp = tempfile::tempdir().unwrap();
14734        let mut runner = registered_runner(tmp.path()).await;
14735        runner.state.config.graph.timeout_resource_wait = 1;
14736        let winner = runner.state.candidates[0].clone();
14737        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14738        let leftover = winner.worktree.join("leftover");
14739        std::fs::write(&leftover, "untrusted edits").unwrap();
14740        let next = tmp.path().join("next-fixer");
14741        let mut failed = spec("first");
14742        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14743        let mut successor = spec("second");
14744        successor.command = vec![
14745            "sh".into(),
14746            "-c".into(),
14747            format!("touch '{}'; echo done", next.display()),
14748        ];
14749        let attempts = vec![
14750            (failed.clone(), "first".into()),
14751            (successor.clone(), "second".into()),
14752        ];
14753        let lock_path = runner.state.repo.join(".git/updater.lock");
14754        let external = std::fs::File::create(&lock_path).unwrap();
14755        assert_eq!(
14756            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14757            0
14758        );
14759        let artifacts = tmp.path().join("artifacts");
14760        let build = |spec, seat| SeatJob {
14761            spec,
14762            seat,
14763            cwd: winner.worktree.clone(),
14764            prompt: "fixture".into(),
14765            timeout: Duration::from_secs(10),
14766            allow_write: true,
14767            sessions: false,
14768            artifacts: artifacts.clone(),
14769            stem: "fix".into(),
14770            handover: None,
14771        };
14772        let (_, _, out) = runner
14773            .ask_fixer_among(
14774                attempts.clone(),
14775                "graft",
14776                None,
14777                Some((&winner.worktree, &head)),
14778                build,
14779            )
14780            .await;
14781        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14782        assert!(leftover.exists(), "contention must prevent clean");
14783        assert!(!next.exists());
14784        assert!(runner.state.handovers.is_empty());
14785        // Early interrupted-repair rollback uses the same protected route.
14786        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14787            from_run: "earlier".into(),
14788            generation: 1,
14789            mode: crate::recovery::Mode::VerifyRepair,
14790            reasons: Vec::new(),
14791            reviewed_head: None,
14792            rounds: Vec::new(),
14793            open_findings: Vec::new(),
14794            evidence: None,
14795            criteria_inherited: true,
14796            repair: crate::recovery::Repair::Started {
14797                before: head.clone(),
14798            },
14799        });
14800        assert!(
14801            runner
14802                .recovery_repair(&winner)
14803                .await
14804                .unwrap_err()
14805                .is::<crate::repo_lock::ResourceBlocked>()
14806        );
14807        assert!(matches!(
14808            runner.state.recovery.as_ref().unwrap().repair,
14809            crate::recovery::Repair::Started { .. }
14810        ));
14811        assert!(leftover.exists());
14812        drop(external);
14813        runner.recovery_repair(&winner).await.unwrap();
14814        assert!(!leftover.exists());
14815        assert!(matches!(
14816            runner.state.recovery.as_ref().unwrap().repair,
14817            crate::recovery::Repair::Done { .. }
14818        ));
14819        // A hard lock-open failure also leaves HEAD and untracked work intact.
14820        std::fs::remove_file(&lock_path).unwrap();
14821        std::fs::create_dir(&lock_path).unwrap();
14822        std::fs::write(&leftover, "untrusted edits").unwrap();
14823        let (_, _, out) = runner
14824            .ask_fixer_among(
14825                attempts,
14826                "graft",
14827                None,
14828                Some((&winner.worktree, &head)),
14829                build,
14830            )
14831            .await;
14832        assert!(matches!(out, AgentOutcome::Failed(_)));
14833        assert!(leftover.exists());
14834        assert!(!next.exists());
14835        assert_eq!(
14836            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14837            head
14838        );
14839    }
14840
14841    #[cfg(unix)]
14842    async fn registered_runner(root: &Path) -> Runner {
14843        repository_runner(root, crate::config::test_coordination()).await
14844    }
14845
14846    async fn repository_runner(
14847        root: &Path,
14848        coordination: Option<crate::config::Coordination>,
14849    ) -> Runner {
14850        ask_test_home();
14851        let repo = root.join("repo");
14852        std::fs::create_dir_all(&repo).unwrap();
14853        init_repo(&repo);
14854        if coordination.is_some() {
14855            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14856        }
14857        let mut runner = runner_at(RunStatus::Prep);
14858        runner.state.repo = repo.clone();
14859        runner.state.config.coordination = coordination;
14860        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14861        runner.state.config.graph.worktree_root = Some(root.join("wt"));
14862        runner.state.config.graph.advise = false;
14863        runner.roles.implementers = vec![spec("implement")];
14864        runner.roles.judges = vec![spec("judge")];
14865        runner.prep().await.unwrap();
14866        runner
14867    }
14868
14869    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14870
14871    #[tokio::test]
14872    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14873        let tmp = tempfile::tempdir().unwrap();
14874        let mut runner = repository_runner(tmp.path(), None).await;
14875        let prompts = runner.state.config.prompts.clone();
14876        let run = runner.state.id.clone();
14877        let ctx = WaveCtx {
14878            carry_seats: false,
14879            pauses: None,
14880            run: &run,
14881            node: "advise",
14882            prompts: &prompts,
14883            cache: None,
14884            round: None,
14885        };
14886        let jobs: Vec<_> = ["alpha", "beta"]
14887            .into_iter()
14888            .map(|id| SeatJob {
14889                spec: scripted(id, AUTH_OUTAGE),
14890                seat: SeatState::new(id, id, 7),
14891                prompt: "propose".into(),
14892                cwd: runner.state.candidates[0].worktree.clone(),
14893                timeout: Duration::from_secs(10),
14894                allow_write: false,
14895                sessions: false,
14896                artifacts: agent::artifacts_dir(&runner.state.dir()),
14897                stem: id.into(),
14898                handover: None,
14899            })
14900            .collect();
14901        let mut losses = Vec::new();
14902        let results = ask_wave_with(
14903            jobs.clone(),
14904            runner.sem.clone(),
14905            2,
14906            &[],
14907            &ctx,
14908            &mut losses,
14909            &mut runner.state,
14910            &|_: &str| Ok(()),
14911        )
14912        .await;
14913        assert!(
14914            results
14915                .iter()
14916                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14917        );
14918        assert_eq!(runner.state.quota.len(), 2);
14919        assert!(
14920            runner
14921                .state
14922                .quota
14923                .iter()
14924                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14925        );
14926        assert!(!runner.state.advise_attempted);
14927        runner.stop_for_resource().unwrap();
14928        assert_eq!(runner.state.status, RunStatus::Blocked);
14929        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14930        assert_eq!(
14931            runner.state.resource_block.as_ref().unwrap().kind,
14932            "agent authentication"
14933        );
14934        // A machine loss followed by an ordinary failure is not evidence
14935        // that the whole node was unavailable, and must not earn a refund.
14936        runner.state.resource_block = None;
14937        runner.state.quota.clear();
14938        let mut mixed = jobs;
14939        mixed[1].spec.command = vec!["false".into()];
14940        let results = ask_wave_with(
14941            mixed,
14942            runner.sem.clone(),
14943            0,
14944            &[],
14945            &ctx,
14946            &mut losses,
14947            &mut runner.state,
14948            &|_: &str| Ok(()),
14949        )
14950        .await;
14951        assert!(results.iter().all(|(_, res, _)| res.is_err()));
14952        assert!(runner.state.resource_block.is_none());
14953        assert_eq!(losses.len(), 1);
14954    }
14955
14956    #[tokio::test]
14957    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14958        let tmp = tempfile::tempdir().unwrap();
14959        let mut runner = repository_runner(tmp.path(), None).await;
14960        let implementer = scripted("implement", AUTH_OUTAGE);
14961        runner.roles.implementers = vec![implementer.clone()];
14962        runner.state.config.agents = vec![implementer];
14963        runner.implement().await.unwrap();
14964        assert_eq!(runner.state.status, RunStatus::Blocked);
14965        assert_eq!(runner.state.quota.len(), 1);
14966        assert_eq!(
14967            runner.state.quota[0].cause,
14968            agent::QuotaCause::Authentication
14969        );
14970        let candidate = &runner.state.candidates[0];
14971        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14972        assert_eq!(runner.state.seats["impl-A"].turns, 1);
14973    }
14974
14975    #[tokio::test]
14976    #[cfg(unix)]
14977    async fn parked_implementer_persists_completed_work() {
14978        for registered in [false, true] {
14979            let tmp = tempfile::tempdir().unwrap();
14980            let mut runner = registered_runner(tmp.path()).await;
14981            if !registered {
14982                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14983            }
14984            let marker = tmp.path().join("started");
14985            let release = tmp.path().join("release");
14986            let mut implementer = spec("implement");
14987            implementer.command = vec![
14988                "sh".into(),
14989                "-c".into(),
14990                format!(
14991                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14992                    marker.display(),
14993                    release.display()
14994                ),
14995            ];
14996            runner.roles.implementers = vec![implementer.clone()];
14997            runner.state.config.agents = vec![implementer];
14998            let pause = runner.pause.clone();
14999            let (result, ()) = tokio::join!(runner.implement(), async {
15000                tokio::time::timeout(Duration::from_secs(10), async {
15001                    while !marker.exists() {
15002                        tokio::time::sleep(Duration::from_millis(10)).await;
15003                    }
15004                })
15005                .await
15006                .unwrap();
15007                pause.park();
15008                std::fs::write(&release, "go").unwrap();
15009            });
15010            result.unwrap();
15011            let candidate = &runner.state.candidates[0];
15012            assert!(candidate.commits > 0, "{candidate:?}");
15013            assert!(candidate.summary.contains("saved-result"));
15014            assert_eq!(candidate.files, 1);
15015            let before = candidate.commits;
15016            runner.implement().await.unwrap();
15017            assert_eq!(runner.state.candidates[0].commits, before);
15018        }
15019    }
15020
15021    #[tokio::test]
15022    #[cfg(unix)]
15023    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
15024        let tmp = tempfile::tempdir().unwrap();
15025        let mut runner = registered_runner(tmp.path()).await;
15026        let winner = runner.state.candidates[0].clone();
15027        let cache = tmp.path().join("cache");
15028        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
15029        runner.state.config.graph.timeout_resource_wait = 0;
15030        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
15031        let crate::cache::AcquireOutcome::Acquired(held) =
15032            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
15033        else {
15034            panic!("cache must be free")
15035        };
15036        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15037            from_run: "earlier".into(),
15038            generation: 1,
15039            mode: crate::recovery::Mode::VerifyRepair,
15040            reasons: Vec::new(),
15041            reviewed_head: None,
15042            rounds: Vec::new(),
15043            open_findings: Vec::new(),
15044            evidence: None,
15045            criteria_inherited: true,
15046            repair: crate::recovery::Repair::NotStarted,
15047        });
15048        runner.recovery_repair(&winner).await.unwrap();
15049        assert!(matches!(
15050            runner.state.recovery.as_ref().unwrap().repair,
15051            crate::recovery::Repair::WaitingForRestore { .. }
15052        ));
15053        drop(held);
15054        runner.state.resource_block = None;
15055        runner.state.config.graph.timeout_resource_wait = 1;
15056        runner.recovery_repair(&winner).await.unwrap();
15057        assert!(matches!(
15058            runner.state.recovery.as_ref().unwrap().repair,
15059            crate::recovery::Repair::Done { .. }
15060        ));
15061        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
15062    }
15063
15064    #[tokio::test]
15065    #[cfg(unix)]
15066    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
15067        use std::os::fd::AsRawFd as _;
15068        for graft in [false, true] {
15069            let tmp = tempfile::tempdir().unwrap();
15070            let mut runner = registered_runner(tmp.path()).await;
15071            runner.state.config.graph.timeout_resource_wait = 1;
15072            let winner = runner.state.candidates[0].clone();
15073            let marker = tmp.path().join("fixer-started");
15074            let mut fixer = spec("fixer");
15075            fixer.command = vec![
15076                "sh".into(),
15077                "-c".into(),
15078                format!("echo started >> '{}'; echo done", marker.display()),
15079            ];
15080            runner.roles.fixer = Some(vec![fixer]);
15081            if graft {
15082                let mut loser = winner.clone();
15083                loser.label = 'B';
15084                loser.index = 1;
15085                runner.state.candidates.push(loser);
15086                runner.state.config.graph.graft_runner_up = true;
15087                runner.state.tally = Some(Tally {
15088                    first_choice: BTreeMap::new(),
15089                    borda: BTreeMap::new(),
15090                    winner: winner.label,
15091                    rankings: 1,
15092                    unanimous_initial: true,
15093                    deliberated: false,
15094                    changed_votes: 0,
15095                    unanimous_final: true,
15096                    tie_break: None,
15097                    judges: 1,
15098                    present: 1,
15099                    quorum: 1,
15100                    met_quorum: true,
15101                    uncontested: None,
15102                });
15103                runner.state.judgements.push(crate::run::Judgement {
15104                    judge: 1,
15105                    seat: "judge-1".into(),
15106                    agent: "judge".into(),
15107                    ranking: vec![winner.label, 'B'],
15108                    reasons: BTreeMap::new(),
15109                    confidence: None,
15110                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
15111                    order: vec![0, 1],
15112                    failed: None,
15113                    duration_ms: 0,
15114                });
15115            } else {
15116                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15117                    from_run: "earlier".into(),
15118                    generation: 1,
15119                    mode: crate::recovery::Mode::VerifyRepair,
15120                    reasons: Vec::new(),
15121                    reviewed_head: None,
15122                    rounds: Vec::new(),
15123                    open_findings: Vec::new(),
15124                    evidence: None,
15125                    criteria_inherited: true,
15126                    repair: crate::recovery::Repair::NotStarted,
15127                });
15128            }
15129            let reader = crate::repo_lock::SeatLease::for_checkout(
15130                &winner.worktree,
15131                false,
15132                crate::config::test_coordination().as_ref(),
15133            )
15134            .await
15135            .unwrap();
15136            let pause = runner.pause.clone();
15137            let (result, ()) = tokio::join!(
15138                async {
15139                    if graft {
15140                        runner.graft().await
15141                    } else {
15142                        runner.recovery_repair(&winner).await
15143                    }
15144                },
15145                async {
15146                    tokio::time::sleep(Duration::from_millis(100)).await;
15147                    pause.park();
15148                }
15149            );
15150            result.unwrap();
15151            assert!(!marker.exists(), "a blocked seat must not spawn");
15152            // Exercise the persisted state, as a binary replacement/resume does.
15153            runner.state = RunState::load(&runner.state.id).unwrap();
15154            if graft {
15155                assert_eq!(
15156                    runner.state.graft.as_ref().unwrap().outcome,
15157                    GraftOutcome::WaitingForRestore
15158                );
15159            } else {
15160                assert!(matches!(
15161                    runner.state.recovery.as_ref().unwrap().repair,
15162                    crate::recovery::Repair::WaitingForRestore { .. }
15163                ));
15164            }
15165            drop(reader);
15166            runner.pause = Pause::new();
15167            runner.state.resource_block = None;
15168            // Deferred restoration must still respect an external lock owner.
15169            let external =
15170                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15171            assert_eq!(
15172                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15173                0
15174            );
15175            let leftover = winner.worktree.join("leftover");
15176            std::fs::write(&leftover, "earlier fallback edits").unwrap();
15177            let result = if graft {
15178                runner.graft().await
15179            } else {
15180                runner.recovery_repair(&winner).await
15181            };
15182            assert!(
15183                result
15184                    .unwrap_err()
15185                    .is::<crate::repo_lock::ResourceBlocked>()
15186            );
15187            assert!(leftover.exists());
15188            assert!(!marker.exists());
15189            drop(external);
15190            runner.state.resource_block = None;
15191            let result = if graft {
15192                runner.graft().await
15193            } else {
15194                runner.recovery_repair(&winner).await
15195            };
15196            result.unwrap();
15197            assert!(!leftover.exists(), "restore must precede retry");
15198            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15199            if graft {
15200                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15201            } else {
15202                assert!(matches!(
15203                    runner.state.recovery.as_ref().unwrap().repair,
15204                    crate::recovery::Repair::Done { .. }
15205                ));
15206            }
15207        }
15208    }
15209
15210    #[tokio::test]
15211    #[cfg(unix)]
15212    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15213        use std::os::fd::AsRawFd as _;
15214        let tmp = tempfile::tempdir().unwrap();
15215        let mut runner = registered_runner(tmp.path()).await;
15216        runner.state.config.graph.timeout_resource_wait = 1;
15217        let winner = runner.state.candidates[0].clone();
15218        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15219        runner.state.tally = Some(Tally {
15220            first_choice: BTreeMap::new(),
15221            borda: BTreeMap::new(),
15222            winner: winner.label,
15223            rankings: 0,
15224            unanimous_initial: false,
15225            deliberated: false,
15226            changed_votes: 0,
15227            unanimous_final: false,
15228            tie_break: None,
15229            judges: 0,
15230            present: 0,
15231            quorum: 0,
15232            met_quorum: true,
15233            uncontested: Some("fixture".into()),
15234        });
15235        runner.state.graft = Some(GraftRecord {
15236            pre: pre.clone(),
15237            post: None,
15238            lines: None,
15239            limit: None,
15240            agent: None,
15241            outcome: GraftOutcome::Running,
15242            requested: Vec::new(),
15243            adoption: None,
15244            adoption_error: None,
15245        });
15246        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15247        git::commit_all(&winner.worktree, "unfinished graft")
15248            .await
15249            .unwrap();
15250        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15251        let leftover = winner.worktree.join("leftover");
15252        std::fs::write(&leftover, "untrusted edits").unwrap();
15253        let path = runner.state.repo.join(".git/updater.lock");
15254        let external = std::fs::File::create(&path).unwrap();
15255        assert_eq!(
15256            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15257            0
15258        );
15259        assert!(
15260            runner
15261                .graft()
15262                .await
15263                .unwrap_err()
15264                .is::<crate::repo_lock::ResourceBlocked>()
15265        );
15266        assert!(matches!(
15267            runner.state.graft.as_ref().unwrap().outcome,
15268            GraftOutcome::Running
15269        ));
15270        assert_eq!(
15271            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15272            unfinished
15273        );
15274        assert!(leftover.exists());
15275        drop(external);
15276        std::fs::remove_file(&path).unwrap();
15277        std::fs::create_dir(&path).unwrap();
15278        assert!(runner.graft().await.is_err());
15279        assert!(matches!(
15280            runner.state.graft.as_ref().unwrap().outcome,
15281            GraftOutcome::Running
15282        ));
15283        assert_eq!(
15284            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15285            unfinished
15286        );
15287        assert!(leftover.exists());
15288        std::fs::remove_dir(&path).unwrap();
15289        runner.graft().await.unwrap();
15290        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15291        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15292        assert!(!leftover.exists());
15293    }
15294
15295    #[tokio::test]
15296    #[cfg(unix)]
15297    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15298        use std::os::fd::AsRawFd as _;
15299        let tmp = tempfile::tempdir().unwrap();
15300        let mut runner = registered_runner(tmp.path()).await;
15301        runner.state.prep_complete = Some(false);
15302        runner.state.config.graph.timeout_resource_wait = 1;
15303        runner.state.save().unwrap();
15304        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15305        assert_eq!(
15306            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15307            0
15308        );
15309        runner.execute().await.unwrap();
15310        let disk = RunState::load(&runner.state.id).unwrap();
15311        assert!(disk.driver_exited);
15312        assert_eq!(disk.driver_pid, None);
15313        assert_eq!(runner.state.status, RunStatus::Blocked);
15314        assert_eq!(
15315            runner.state.resource_block.as_ref().unwrap().kind,
15316            "repository coordination"
15317        );
15318        assert!(runner.state.prep_failure.is_none());
15319        assert!(runner.state.seats.is_empty());
15320        runner.pause.park();
15321        // Include headroom for Git subprocesses under parallel test load;
15322        // the repository contention deadline itself remains one second.
15323        tokio::time::timeout(Duration::from_secs(10), runner.execute())
15324            .await
15325            .unwrap()
15326            .unwrap();
15327        assert!(runner.state.parked);
15328        assert!(runner.state.prep_failure.is_none());
15329        let disk = RunState::load(&runner.state.id).unwrap();
15330        assert!(disk.driver_exited);
15331        assert_eq!(disk.driver_pid, None);
15332    }
15333
15334    #[tokio::test]
15335    #[cfg(unix)]
15336    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15337        use std::os::fd::AsRawFd as _;
15338        let tmp = tempfile::tempdir().unwrap();
15339        let mut runner = registered_runner(tmp.path()).await;
15340        runner.state.prep_complete = Some(false);
15341        runner.state.config.graph.timeout_resource_wait = 30;
15342        runner.state.save().unwrap();
15343        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15344        assert_eq!(
15345            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15346            0
15347        );
15348        assert!(
15349            tokio::time::timeout(Duration::from_millis(100), runner.execute())
15350                .await
15351                .is_err()
15352        );
15353        let disk = RunState::load(&runner.state.id).unwrap();
15354        assert!(disk.driver_exited);
15355        assert_eq!(disk.driver_pid, None);
15356        assert!(disk.status.resumable());
15357        assert!(runner.state.driver_exited);
15358        assert_eq!(runner.state.driver_pid, None);
15359        runner.state.save().unwrap();
15360        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15361    }
15362
15363    #[tokio::test]
15364    #[cfg(unix)]
15365    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15366        let tmp = tempfile::tempdir().unwrap();
15367        let mut runner = registered_runner(tmp.path()).await;
15368        let winner = runner.state.candidates[0].clone();
15369        let cache = tmp.path().join("cache");
15370        let started = tmp.path().join("pre-gate-started");
15371        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15372        runner.state.config.verify.gate =
15373            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15374        let home = crate::run::home();
15375        let owner =
15376            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15377        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15378            crate::cache::AcquireOutcome::Acquired(g) => g,
15379            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15380        };
15381        let mut gate = Box::pin(runner.run_gate(&winner));
15382        assert!(
15383            tokio::time::timeout(Duration::from_millis(150), &mut gate)
15384                .await
15385                .is_err()
15386        );
15387        assert!(!started.exists(), "pre-gate must wait for the cache too");
15388        // Simulate the wave that already owns the cache entering invoke.
15389        // It must obtain the editor reservation and finish without waiting
15390        // for the gate that is queued behind its cache.
15391        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15392        let editor = crate::repo_lock::SeatLease::checkout_with(
15393            &winner.worktree,
15394            true,
15395            crate::config::test_coordination().as_ref(),
15396            &wait,
15397        )
15398        .await
15399        .unwrap();
15400        drop(editor);
15401        guard.release();
15402        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15403            .await
15404            .unwrap()
15405            .unwrap();
15406        drop(gate);
15407        assert!(outcomes.iter().all(CommandOutcome::ok));
15408        assert!(started.exists());
15409    }
15410
15411    #[tokio::test]
15412    #[cfg(unix)]
15413    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15414        use std::os::fd::AsRawFd as _;
15415
15416        ask_test_home();
15417        let tmp = tempfile::tempdir().unwrap();
15418        let repo = tmp.path().join("repo");
15419        std::fs::create_dir_all(&repo).unwrap();
15420        init_repo(&repo);
15421        let common = repo.join(".git");
15422        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15423        let mut runner = runner_at(RunStatus::Prep);
15424        runner.state.repo = repo.clone();
15425        runner.state.config.coordination = crate::config::test_coordination();
15426        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15427        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15428        runner.state.config.graph.advise = false;
15429        runner.roles.implementers = vec![spec("implement")];
15430        runner.roles.judges = vec![spec("judge")];
15431        runner.prep().await.unwrap();
15432        let winner = runner.state.candidates[0].clone();
15433        // Emulate edits already left by a completed seat. The fixture CLI
15434        // only reports completion; it performs no repository operations.
15435        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15436        let completed = tmp.path().join("seat-completed");
15437        runner.roles.implementers[0].command = vec![
15438            "sh".into(),
15439            "-c".into(),
15440            format!("touch '{}'; echo completed", completed.display()),
15441        ];
15442        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15443        assert_eq!(
15444            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15445            0
15446        );
15447        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15448        let mut implement = Box::pin(runner.implement());
15449        tokio::select! {
15450            result = &mut implement => panic!("rescue must wait: {result:?}"),
15451            result = tokio::time::timeout(Duration::from_secs(10), async {
15452                while !completed.exists() {
15453                    tokio::time::sleep(Duration::from_millis(20)).await;
15454                }
15455            }) => result.unwrap(),
15456        }
15457        assert!(
15458            tokio::time::timeout(Duration::from_millis(200), &mut implement)
15459                .await
15460                .is_err()
15461        );
15462        assert_eq!(
15463            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15464            before
15465        );
15466        assert_eq!(
15467            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15468            0
15469        );
15470        tokio::time::timeout(Duration::from_secs(10), &mut implement)
15471            .await
15472            .unwrap()
15473            .unwrap();
15474        drop(implement);
15475        assert_ne!(
15476            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15477            before
15478        );
15479        assert!(git::is_clean(&winner.worktree).await.unwrap());
15480
15481        let started = tmp.path().join("pre-gate-started");
15482        let proceed = tmp.path().join("pre-gate-proceed");
15483        let checked = tmp.path().join("gate-ran");
15484        runner.state.config.verify.pre_gate = vec![format!(
15485            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15486            started.display(),
15487            proceed.display()
15488        )];
15489        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15490        assert_eq!(
15491            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15492            0
15493        );
15494        let mut gate = Box::pin(runner.run_gate(&winner));
15495        assert!(
15496            tokio::time::timeout(Duration::from_millis(200), &mut gate)
15497                .await
15498                .is_err()
15499        );
15500        assert!(!started.exists());
15501        assert!(!checked.exists());
15502        assert!(git::is_clean(&winner.worktree).await.unwrap());
15503        assert_eq!(
15504            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15505            0
15506        );
15507        tokio::select! {
15508            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15509            result = tokio::time::timeout(Duration::from_secs(10), async {
15510                while !started.exists() {
15511                    tokio::time::sleep(Duration::from_millis(20)).await;
15512                }
15513            }) => result.unwrap(),
15514        }
15515        assert_eq!(
15516            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15517            -1,
15518            "mechanical edits must hold the real lock"
15519        );
15520        std::fs::write(proceed, "ready").unwrap();
15521        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15522            .await
15523            .unwrap()
15524            .unwrap();
15525        drop(gate);
15526        assert!(outcomes.iter().all(CommandOutcome::ok));
15527        assert!(checked.exists());
15528        assert!(runner.state.pre_gate_commit.is_some());
15529        assert!(git::is_clean(&winner.worktree).await.unwrap());
15530        assert_eq!(
15531            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15532            0,
15533            "gate completion must release coordination"
15534        );
15535        drop(outsider);
15536        std::fs::remove_file(common.join("updater.lock")).unwrap();
15537        std::fs::create_dir(common.join("updater.lock")).unwrap();
15538        std::fs::remove_file(&started).unwrap();
15539        std::fs::remove_file(&checked).unwrap();
15540        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15541        let error = runner.run_gate(&winner).await.unwrap_err();
15542        assert!(
15543            error
15544                .to_string()
15545                .contains("acquire mandatory repository lock")
15546        );
15547        assert!(
15548            !started.exists(),
15549            "failed acquisition must prevent pre-gate"
15550        );
15551        assert!(!checked.exists(), "failed acquisition must prevent gate");
15552        assert!(git::is_clean(&winner.worktree).await.unwrap());
15553        assert_eq!(
15554            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15555            head
15556        );
15557    }
15558
15559    #[tokio::test]
15560    #[cfg(unix)]
15561    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15562        use std::os::fd::AsRawFd as _;
15563        use std::time::Duration;
15564
15565        ask_test_home();
15566        let tmp = tempfile::tempdir().unwrap();
15567        let repo = tmp.path().join("repo");
15568        std::fs::create_dir_all(&repo).unwrap();
15569        init_repo(&repo);
15570        let common = repo.join(".git");
15571        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15572        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15573        assert_eq!(
15574            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15575            0
15576        );
15577        let mut runner = runner_at(RunStatus::Prep);
15578        runner.state.repo = repo.clone();
15579        runner.state.config.coordination = crate::config::test_coordination();
15580        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15581        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15582        runner.state.config.graph.advise = true;
15583        runner.state.config.graph.advisors = 1;
15584        runner.roles.implementers = vec![spec("implement")];
15585        runner.roles.judges = vec![spec("judge")];
15586        let root = runner.state.worktree_root();
15587        let started = tmp.path().join("setup-started");
15588        let proceed = tmp.path().join("setup-proceed");
15589        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15590            run: Some(format!(
15591                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15592                started.display(),
15593                proceed.display()
15594            )),
15595            ..Default::default()
15596        }];
15597        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15598        let config = git::git(&repo, &["config", "--local", "--list"])
15599            .await
15600            .unwrap();
15601        let mut prep = Box::pin(runner.prep());
15602        assert!(
15603            tokio::time::timeout(Duration::from_millis(200), &mut prep)
15604                .await
15605                .is_err()
15606        );
15607        assert!(
15608            !root.exists(),
15609            "contention must prevent even checkout creation"
15610        );
15611        assert!(!started.exists(), "contention must prevent setup");
15612        assert_eq!(
15613            git::git(&repo, &["branch", "--list"]).await.unwrap(),
15614            branches
15615        );
15616        assert_eq!(
15617            git::git(&repo, &["config", "--local", "--list"])
15618                .await
15619                .unwrap(),
15620            config
15621        );
15622        drop(lock);
15623        tokio::select! {
15624            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15625            result = tokio::time::timeout(Duration::from_secs(10), async {
15626                while !started.exists() {
15627                    tokio::time::sleep(Duration::from_millis(20)).await;
15628                }
15629            }) => result.unwrap(),
15630        }
15631        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15632        assert_eq!(
15633            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15634            -1
15635        );
15636        assert!(
15637            tokio::time::timeout(
15638                Duration::from_millis(100),
15639                crate::repo_lock::SeatLease::for_checkout(
15640                    &repo,
15641                    true,
15642                    crate::config::test_coordination().as_ref()
15643                )
15644            )
15645            .await
15646            .is_err(),
15647            "preparation must also exclude local editors"
15648        );
15649        std::fs::write(proceed, "ready").unwrap();
15650        tokio::time::timeout(Duration::from_secs(10), &mut prep)
15651            .await
15652            .unwrap()
15653            .unwrap();
15654        drop(prep);
15655        assert_eq!(runner.state.prep_complete, Some(true));
15656        assert!(root.join("judge-1").exists());
15657        assert!(root.join("advisor-1").exists());
15658        assert_eq!(
15659            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15660            0
15661        );
15662    }
15663
15664    #[tokio::test]
15665    #[cfg(unix)]
15666    async fn preparation_lock_failure_prevents_repository_changes() {
15667        ask_test_home();
15668        let tmp = tempfile::tempdir().unwrap();
15669        let repo = tmp.path().join("repo");
15670        std::fs::create_dir_all(&repo).unwrap();
15671        init_repo(&repo);
15672        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15673        // A directory at the required lock path makes opening the real lock
15674        // fail, rather than merely exercising an invalid registration.
15675        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15676        let mut runner = runner_at(RunStatus::Prep);
15677        runner.state.repo = repo.clone();
15678        runner.state.config.coordination = crate::config::test_coordination();
15679        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15680        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15681        runner.roles.implementers = vec![spec("implement")];
15682        let error = runner.prep().await.unwrap_err();
15683        assert!(
15684            error
15685                .to_string()
15686                .contains("acquire mandatory repository lock")
15687        );
15688        assert!(!runner.state.worktree_root().exists());
15689        assert!(runner.state.candidates.is_empty());
15690        assert!(!repo.join(".git/config.worktree").exists());
15691        assert_eq!(
15692            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15693            "* main"
15694        );
15695    }
15696
15697    #[tokio::test]
15698    async fn preparation_records_creation_intent_before_git_can_fail() {
15699        ask_test_home();
15700        let tmp = tempfile::tempdir().unwrap();
15701        let repo = tmp.path().join("repo");
15702        std::fs::create_dir_all(&repo).unwrap();
15703        init_repo(&repo);
15704        let mut runner = runner_at(RunStatus::Prep);
15705        runner.state.repo = repo.clone();
15706        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15707        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15708        runner.state.config.blind.commit_msg_hook = false;
15709        runner.roles.implementers = vec![spec("one")];
15710        let label = blind::assign_labels(1, runner.state.seed)[0];
15711        let branch = runner.state.branch_for(label);
15712        // A conflicting branch makes Git refuse creation before a candidate
15713        // can be registered. The durable intent must still identify it.
15714        git::git(&repo, &["branch", &branch]).await.unwrap();
15715        assert!(runner.prep().await.is_err());
15716        let saved = RunState::load(&runner.state.id).unwrap();
15717        assert!(saved.candidates.is_empty());
15718        let event = saved
15719            .events
15720            .iter()
15721            .find(|e| e.node == "prep-intent")
15722            .unwrap();
15723        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15724        assert_eq!(intent["branch"], branch);
15725        assert_eq!(intent["start"], runner.state.base_commit);
15726        assert_eq!(
15727            intent["worktree"],
15728            runner
15729                .state
15730                .worktree_root()
15731                .join(format!("cand-{label}"))
15732                .to_string_lossy()
15733                .as_ref()
15734        );
15735        assert_eq!(intent["picks"], serde_json::json!([]));
15736    }
15737
15738    #[tokio::test]
15739    async fn invalid_pinned_sources_create_no_partial_worktrees() {
15740        ask_test_home();
15741        let tmp = tempfile::tempdir().unwrap();
15742        let repo = tmp.path().join("repo");
15743        std::fs::create_dir_all(&repo).unwrap();
15744        init_repo(&repo);
15745        let mut runner = runner_at(RunStatus::Prep);
15746        runner.state.repo = repo.clone();
15747        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15748        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15749        runner.state.sources.pick_commits.push(refs::Input {
15750            token: "missing".into(),
15751            sha: "0".repeat(40),
15752            provenance: refs::Provenance::ExplicitCli,
15753        });
15754        assert!(runner.prep().await.is_err());
15755        assert!(runner.state.candidates.is_empty());
15756        assert!(!runner.state.worktree_root().exists());
15757    }
15758
15759    #[tokio::test]
15760    async fn prose_and_legacy_inference_never_import_into_candidates() {
15761        ask_test_home();
15762        let tmp = tempfile::tempdir().unwrap();
15763        let repo = tmp.path().join("repo");
15764        std::fs::create_dir_all(&repo).unwrap();
15765        init_repo(&repo);
15766        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15767        git::git(&repo, &["checkout", "-b", "magi/history/A"])
15768            .await
15769            .unwrap();
15770        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15771        git::git(&repo, &["add", "."]).await.unwrap();
15772        git::git(&repo, &["commit", "-m", "historical implementation"])
15773            .await
15774            .unwrap();
15775        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15776        git::git(&repo, &["checkout", "main"]).await.unwrap();
15777        for instruction in [
15778            format!("Example: magi/history/A at {source}"),
15779            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15780            format!("Never import magi/history/A or {source}"),
15781            format!("Conversation answer: I mentioned {source} only as background"),
15782        ] {
15783            let mut runner = runner_at(RunStatus::Prep);
15784            runner.state.repo = repo.clone();
15785            runner.state.base_commit = base.clone();
15786            runner.state.instruction = instruction.clone();
15787            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15788            runner.state.config.blind.commit_msg_hook = false;
15789            runner.state.seeds = vec![refs::Seed {
15790                token: "magi/history/A".into(),
15791                sha: source.clone(),
15792                kind: refs::SeedKind::Unmerged,
15793                branch: true,
15794                detail: String::new(),
15795            }];
15796            runner.roles.implementers = vec![spec("one")];
15797            assert_eq!(seeded_instruction(&runner.state), instruction);
15798            // Model an old record with inferred seeds but no structured input field.
15799            let mut legacy = serde_json::to_value(&runner.state).unwrap();
15800            legacy.as_object_mut().unwrap().remove("sources");
15801            runner.state = serde_json::from_value(legacy).unwrap();
15802            assert_eq!(runner.state.sources, refs::Sources::default());
15803            runner.state.save().unwrap();
15804            runner.state = RunState::load(&runner.state.id).unwrap();
15805            runner.prep().await.unwrap();
15806            for candidate in &runner.state.candidates {
15807                assert_eq!(
15808                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15809                    base
15810                );
15811                assert!(!candidate.worktree.join("unwanted").exists());
15812            }
15813        }
15814    }
15815
15816    #[tokio::test]
15817    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15818        ask_test_home();
15819        let tmp = tempfile::tempdir().unwrap();
15820        let repo = tmp.path().join("repo");
15821        std::fs::create_dir_all(&repo).unwrap();
15822        init_repo(&repo);
15823        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15824        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15825        std::fs::write(repo.join("referenced"), "required commit").unwrap();
15826        git::git(&repo, &["add", "."]).await.unwrap();
15827        git::git(&repo, &["commit", "-m", "required seed"])
15828            .await
15829            .unwrap();
15830        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15831        git::git(&repo, &["checkout", "main"]).await.unwrap();
15832        let mut runner = runner_at(RunStatus::Prep);
15833        runner.state.repo = repo.clone();
15834        runner.state.base_commit = base.clone();
15835        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15836        runner.state.config.graph.advise = false;
15837        runner.state.config.blind.commit_msg_hook = false;
15838        runner.state.sources =
15839            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15840                .await
15841                .unwrap();
15842        runner.roles.implementers = vec![spec("one"), spec("two")];
15843        runner.roles.judges = vec![spec("judge")];
15844        let label = blind::assign_labels(2, runner.state.seed)[0];
15845        let branch = runner.state.branch_for(label);
15846        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15847        git::worktree_add_branch(&repo, &worktree, &branch, &base)
15848            .await
15849            .unwrap();
15850        runner.state.candidates.push(Candidate {
15851            index: 0,
15852            label,
15853            agent: "one".into(),
15854            branch,
15855            worktree,
15856            summary: String::new(),
15857            stat: String::new(),
15858            files: 0,
15859            commits: 0,
15860            empty: false,
15861            failed: None,
15862            verified_noop: None,
15863            duration_ms: 0,
15864            folded: false,
15865        });
15866        runner.state.save().unwrap();
15867        // Reload the checkpoint immediately before its first cherry-pick.
15868        runner.state = RunState::load(&runner.state.id).unwrap();
15869        assert_eq!(runner.state.prep_complete, Some(false));
15870        runner.prep().await.unwrap();
15871        assert_eq!(runner.state.prep_complete, Some(true));
15872        assert_eq!(runner.state.candidates.len(), 2);
15873        for candidate in &runner.state.candidates {
15874            assert_eq!(
15875                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15876                git::tree_of(&repo, &seed).await.unwrap()
15877            );
15878            assert_eq!(
15879                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15880                "required commit"
15881            );
15882        }
15883        assert!(runner.state.worktree_root().join("judge-1").exists());
15884        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15885        runner.prep().await.unwrap();
15886        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15887
15888        runner.state.prep_complete = Some(false);
15889        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15890            run: Some(format!(
15891                "test -f '{}'",
15892                tmp.path().join("dependency-ready").display()
15893            )),
15894            ..Default::default()
15895        }];
15896        assert!(runner.prep().await.is_err());
15897        assert!(runner.state.prep_failure.is_some());
15898        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15899        runner.state = RunState::load(&runner.state.id).unwrap();
15900        assert!(
15901            runner.prep().await.is_err(),
15902            "correcting the cause alone must not auto-retry"
15903        );
15904        let failed_at = runner.state.events.last().unwrap().at;
15905        assert!(
15906            runner.retry_preparation(failed_at).is_err(),
15907            "old approvals cannot unlock the hold"
15908        );
15909        let mut task = crate::queue::Task::new(
15910            "retry".into(),
15911            "retry setup".into(),
15912            repo.clone(),
15913            crate::queue::Source::Human,
15914        );
15915        task.start(runner.state.id.clone());
15916        task.hold_machine(Some("setup failed".into()));
15917        task.release_operator("cli", None);
15918        runner
15919            .retry_preparation(task.resumption.as_ref().unwrap().at)
15920            .unwrap();
15921        runner.prep().await.unwrap();
15922        assert!(runner.state.prep_failure.is_none());
15923        assert_eq!(runner.state.prep_complete, Some(true));
15924        assert_eq!(runner.state.candidates.len(), 2);
15925        assert!(runner.state.worktree_root().join("judge-1").exists());
15926        for candidate in &runner.state.candidates {
15927            assert_eq!(
15928                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15929                git::tree_of(&repo, &seed).await.unwrap()
15930            );
15931        }
15932    }
15933
15934    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15935        let mut heads = Vec::new();
15936        for candidate in candidates {
15937            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15938        }
15939        heads
15940    }
15941
15942    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15943        ask_test_home();
15944        let tmp = tempfile::tempdir().unwrap();
15945        let repo = tmp.path().join("repo");
15946        std::fs::create_dir(&repo).unwrap();
15947        init_repo(&repo);
15948        let mut runner = runner_at(RunStatus::Reviewing);
15949        runner.state.repo = repo.clone();
15950        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15951        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15952        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15953            let branch = format!("candidate-{label}");
15954            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15955            git::worktree_add_branch(&repo, &worktree, &branch, "main")
15956                .await
15957                .unwrap();
15958            if label == 'B' {
15959                std::fs::write(worktree.join("change"), "winner").unwrap();
15960                git::rescue_commit(&worktree, "candidate change")
15961                    .await
15962                    .unwrap();
15963            }
15964            runner.state.candidates.push(Candidate {
15965                index,
15966                label,
15967                agent: "alpha".into(),
15968                branch,
15969                worktree,
15970                summary: "completed reply".into(),
15971                stat: String::new(),
15972                files: usize::from(label == 'B'),
15973                commits: usize::from(label == 'B'),
15974                empty: label == 'A',
15975                failed: None,
15976                verified_noop: None,
15977                duration_ms: 1,
15978                folded: false,
15979            });
15980        }
15981        runner.state.judge_skipped = true;
15982        runner.state.tally = Some(Tally {
15983            first_choice: BTreeMap::new(),
15984            borda: BTreeMap::new(),
15985            winner: 'B',
15986            rankings: 0,
15987            unanimous_initial: false,
15988            deliberated: false,
15989            changed_votes: 0,
15990            unanimous_final: false,
15991            tie_break: None,
15992            judges: 0,
15993            present: 0,
15994            quorum: 0,
15995            met_quorum: true,
15996            uncontested: Some("only B produced a change".into()),
15997        });
15998        runner.state.save().unwrap();
15999        (tmp, runner)
16000    }
16001
16002    #[tokio::test]
16003    async fn fold_losers_persists_discard_before_later_teardown_failure() {
16004        let (_tmp, mut runner) = preservation_resume_fixture().await;
16005        runner.state.candidates[0].folded = true;
16006        runner.state.config.graph.advise = true;
16007        runner.state.config.graph.advisors = 2;
16008        let repo = runner.state.repo.clone();
16009        let root = runner.state.worktree_root();
16010        let discarded = root.join("advisor-1");
16011        git::worktree_add_detached(&repo, &discarded, "main")
16012            .await
16013            .unwrap();
16014        std::fs::write(discarded.join("design_proposal.json"), "{}").unwrap();
16015        let protected = root.join("advisor-2");
16016        git::worktree_add_branch(&repo, &protected, "attached-advisor", "main")
16017            .await
16018            .unwrap();
16019        std::fs::write(protected.join("keep"), "uncommitted").unwrap();
16020        runner.state.save().unwrap();
16021
16022        assert!(runner.fold_losers().await.is_err());
16023        assert!(!discarded.exists());
16024        assert!(protected.join("keep").exists());
16025        let saved = RunState::load(&runner.state.id).unwrap();
16026        assert!(saved.events.iter().any(|event| {
16027            event.message.contains("advisor-1") && event.message.contains("design_proposal.json")
16028        }));
16029    }
16030
16031    #[tokio::test]
16032    async fn read_seat_teardown_discards_only_owned_detached_seats() {
16033        let (tmp, mut runner) = preservation_resume_fixture().await;
16034        let repo = runner.state.repo.clone();
16035        let root = runner.state.worktree_root();
16036        for name in ["advisor-3", "judge-1", "review-2"] {
16037            let path = root.join(name);
16038            git::worktree_add_detached(&repo, &path, "main")
16039                .await
16040                .unwrap();
16041            std::fs::write(path.join("README.md"), "uncommitted edit").unwrap();
16042            std::fs::write(path.join("design_proposal.json"), "{}").unwrap();
16043            remove_read_seat_worktree(&mut runner.state, &path)
16044                .await
16045                .unwrap();
16046            assert!(!path.exists());
16047        }
16048        let outside = tmp.path().join("advisor-1");
16049        let unknown = root.join("unknown");
16050        let attached = root.join("advisor-4");
16051        git::worktree_add_detached(&repo, &outside, "main")
16052            .await
16053            .unwrap();
16054        git::worktree_add_detached(&repo, &unknown, "main")
16055            .await
16056            .unwrap();
16057        git::worktree_add_branch(&repo, &attached, "attached-advisor", "main")
16058            .await
16059            .unwrap();
16060        let foreign = tmp.path().join("foreign");
16061        std::fs::create_dir(&foreign).unwrap();
16062        init_repo(&foreign);
16063        let wrong_repo = root.join("judge-2");
16064        git::worktree_add_detached(&foreign, &wrong_repo, "main")
16065            .await
16066            .unwrap();
16067        let candidate = runner.state.candidates[0].worktree.clone();
16068        for path in [outside, unknown, attached, wrong_repo, candidate] {
16069            std::fs::write(path.join("uncommitted"), "keep").unwrap();
16070            assert!(
16071                remove_read_seat_worktree(&mut runner.state, &path)
16072                    .await
16073                    .is_err()
16074            );
16075            assert_eq!(
16076                std::fs::read_to_string(path.join("uncommitted")).unwrap(),
16077                "keep"
16078            );
16079        }
16080    }
16081
16082    #[tokio::test]
16083    #[cfg(unix)]
16084    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
16085        for (solo, failed_checkpoint) in
16086            [(true, false), (false, false), (true, true), (false, true)]
16087        {
16088            let (_tmp, mut runner) = preservation_resume_fixture().await;
16089            if solo {
16090                runner.state.candidates.retain(|c| c.label == 'A');
16091            }
16092            runner.state.candidates[0].empty = false;
16093            runner.state.candidates[0].summary.clear();
16094            runner.state.tally = None;
16095            runner.state.judge_skipped = false;
16096            runner.state.status = RunStatus::Implementing;
16097            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16098            let mut implementer = spec("alpha");
16099            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16100            runner.roles.implementers = vec![implementer];
16101            let patch = runner.state.dir().join("artifacts/cand-A.patch");
16102            if failed_checkpoint {
16103                std::fs::create_dir_all(&patch).unwrap();
16104            }
16105            runner.state.save().unwrap();
16106            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
16107            drop(pressure);
16108            runner.state = RunState::load(&runner.state.id).unwrap();
16109            assert_eq!(runner.state.status, RunStatus::Blocked);
16110            assert!(runner.state.interrupted_implementations.contains(&'A'));
16111            assert_eq!(runner.state.candidates[0].commits, 0);
16112            assert!(!runner.state.candidates[0].empty);
16113            assert!(runner.state.candidates[0].failed.is_none());
16114            if failed_checkpoint {
16115                assert!(runner.state.candidate_preservation.contains_key(&'A'));
16116                std::fs::remove_dir(patch).unwrap();
16117            }
16118            runner.state.resource_block = None;
16119            runner.state.config.disk.min_free_bytes = 0;
16120            runner.roles.implementers[0].command = vec![
16121                "sh".into(),
16122                "-c".into(),
16123                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
16124            ];
16125            runner.implement().await.unwrap();
16126            assert!(runner.state.interrupted_implementations.is_empty());
16127            assert!(runner.state.candidate_preservation.is_empty());
16128            assert_eq!(runner.state.candidates[0].commits, 1);
16129            assert!(!runner.state.candidates[0].empty);
16130            assert!(
16131                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
16132                    .unwrap()
16133                    .contains("retried empty interrupted seat")
16134            );
16135            if !solo {
16136                assert_eq!(runner.state.candidates[1].commits, 1);
16137            }
16138        }
16139    }
16140
16141    #[tokio::test]
16142    #[cfg(unix)]
16143    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
16144        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
16145            let (_tmp, mut runner) = preservation_resume_fixture().await;
16146            runner.state.candidates.retain(|c| c.label == 'A');
16147            runner.state.candidates[0].empty = false;
16148            runner.state.candidates[0].summary.clear();
16149            runner.state.tally = None;
16150            runner.state.judge_skipped = false;
16151            runner.state.status = RunStatus::Implementing;
16152            // Force only the real post-spawn watchdog, without failing the
16153            // separate admission check. Existing edits predate its measurement.
16154            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16155            let wt = runner.state.candidates[0].worktree.clone();
16156            std::fs::write(
16157                wt.join("interrupted-source"),
16158                "retain implementation work\n",
16159            )
16160            .unwrap();
16161            if committed {
16162                git::rescue_commit(&wt, "implementation checkpoint")
16163                    .await
16164                    .unwrap();
16165            }
16166            let lock = git::git(
16167                &wt,
16168                &[
16169                    "rev-parse",
16170                    "--path-format=absolute",
16171                    "--git-path",
16172                    "index.lock",
16173                ],
16174            )
16175            .await
16176            .unwrap();
16177            if block_rescue {
16178                std::fs::write(&lock, "fixture-owned lock").unwrap();
16179            }
16180            let mut implementer = spec("alpha");
16181            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16182            runner.roles.implementers = vec![implementer];
16183            runner.state.save().unwrap();
16184            let result = runner.implement().await;
16185            drop(pressure);
16186            assert_eq!(result.is_err(), block_rescue);
16187            assert_eq!(
16188                runner.state.resource_block.as_ref().unwrap().kind,
16189                "disk pressure"
16190            );
16191            assert!(
16192                runner
16193                    .state
16194                    .events
16195                    .iter()
16196                    .any(|e| e.message.contains("agent execution interrupted"))
16197            );
16198            assert!(
16199                runner
16200                    .state
16201                    .dir()
16202                    .join("artifacts/candidate-A-recovery/staged.patch")
16203                    .exists()
16204            );
16205            if block_rescue {
16206                let saved = RunState::load(&runner.state.id).unwrap();
16207                assert!(saved.candidate_preservation.contains_key(&'A'));
16208                assert!(!saved.candidates[0].empty);
16209                assert_eq!(
16210                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16211                    "retain implementation work\n"
16212                );
16213                std::fs::remove_file(lock).unwrap();
16214                runner.state = saved;
16215            } else {
16216                assert_eq!(runner.state.status, RunStatus::Blocked);
16217                assert_eq!(runner.state.candidates[0].commits, 1);
16218            }
16219            // Recovery must save the candidate before deciding whether another
16220            // implementation is needed. No implementer is available on retry.
16221            runner.roles.implementers.clear();
16222            runner.state.config.disk.min_free_bytes = 0;
16223            runner.state.resource_block = None;
16224            runner.implement().await.unwrap();
16225            assert!(runner.state.candidate_preservation.is_empty());
16226            assert_eq!(runner.state.candidates[0].commits, 1);
16227            assert!(!runner.state.candidates[0].empty);
16228            assert!(git::is_clean(&wt).await.unwrap());
16229            assert!(
16230                crate::run::read_artifact(&runner.state, "cand-A.patch")
16231                    .unwrap()
16232                    .contains("retain implementation work")
16233            );
16234        }
16235    }
16236
16237    #[tokio::test]
16238    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16239        let (_tmp, mut runner) = preservation_resume_fixture().await;
16240        // Schema 15 already confirmed the unfolded no-change result.
16241        runner.implement().await.unwrap();
16242        assert!(runner.state.candidate_preservation.is_empty());
16243        assert!(
16244            !runner
16245                .state
16246                .dir()
16247                .join("artifacts/candidate-A-recovery")
16248                .exists()
16249        );
16250        runner.fold_losers().await.unwrap();
16251        assert!(!runner.state.candidates[0].worktree.exists());
16252        for schema in [14, crate::run::SCHEMA] {
16253            runner.state.schema = schema;
16254            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16255            runner.state.save().unwrap();
16256            runner.state = RunState::load(&runner.state.id).unwrap();
16257            runner.implement().await.unwrap();
16258            assert!(runner.state.candidate_preservation.is_empty());
16259            assert!(runner.state.candidates[0].folded);
16260            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16261        }
16262    }
16263
16264    #[tokio::test]
16265    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16266        let (_tmp, mut runner) = preservation_resume_fixture().await;
16267        runner.state.candidates.retain(|c| c.label == 'A');
16268        runner.state.tally = None;
16269        runner.state.judge_skipped = false;
16270        let candidate = &mut runner.state.candidates[0];
16271        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16272        candidate.failed = Some("deployment validation unavailable".into());
16273        std::fs::write(
16274            candidate.worktree.join("repair"),
16275            "future launcher policy\n",
16276        )
16277        .unwrap();
16278        runner
16279            .state
16280            .candidate_preservation
16281            .insert('A', "awaiting persistence".into());
16282        runner.persist_candidates().await.unwrap();
16283        let candidate = &runner.state.candidates[0];
16284        assert!(!candidate.empty);
16285        assert_eq!(candidate.commits, 1);
16286        assert_eq!(candidate.files, 1);
16287        assert!(candidate.failed.is_none());
16288        assert!(candidate.summary.contains("EPERM"));
16289        assert!(
16290            crate::run::read_artifact(&runner.state, "cand-A.patch")
16291                .unwrap()
16292                .contains("future launcher policy")
16293        );
16294        assert!(runner.state.candidate_preservation.is_empty());
16295    }
16296
16297    #[tokio::test]
16298    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16299        let (_tmp, mut runner) = preservation_resume_fixture().await;
16300        let wt = runner.state.candidates[0].worktree.clone();
16301        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16302        runner.state.schema = 14;
16303        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16304        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16305        let tally = serde_json::to_value(&runner.state.tally).unwrap();
16306        for _ in 0..2 {
16307            assert!(runner.implement().await.is_err());
16308            assert_eq!(runner.state.status, RunStatus::Blocked);
16309            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16310            assert!(
16311                runner.state.candidates[0].empty,
16312                "old verdict must remain unusable"
16313            );
16314            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16315            assert!(runner.state.judge_skipped);
16316            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16317            assert!(
16318                crate::run::read_artifact(&runner.state, "cand-A.patch")
16319                    .unwrap()
16320                    .contains("owned edits")
16321            );
16322            assert!(wt.exists());
16323            assert_eq!(
16324                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16325                    .await
16326                    .unwrap(),
16327                1
16328            );
16329            runner.state = RunState::load(&runner.state.id).unwrap();
16330        }
16331    }
16332
16333    #[tokio::test]
16334    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16335        let (_tmp, mut runner) = preservation_resume_fixture().await;
16336        let wt = runner.state.candidates[0].worktree.clone();
16337        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16338        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16339        let index_lock = PathBuf::from(
16340            git::git(
16341                &wt,
16342                &[
16343                    "rev-parse",
16344                    "--path-format=absolute",
16345                    "--git-path",
16346                    "index.lock",
16347                ],
16348            )
16349            .await
16350            .unwrap(),
16351        );
16352        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16353        runner.state.schema = 14;
16354        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16355        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16356        assert!(runner.implement().await.is_err());
16357        assert!(runner.after_implement().is_err());
16358        assert!(runner.fold_losers().await.is_err());
16359        assert_eq!(
16360            std::fs::read_to_string(wt.join("recovered")).unwrap(),
16361            "failed seat edits"
16362        );
16363        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16364        runner.state = RunState::load(&runner.state.id).unwrap();
16365        std::fs::remove_file(index_lock).unwrap();
16366        // Recovery finds changes that the old uncontested result never judged.
16367        assert!(runner.implement().await.is_err());
16368        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16369        assert!(runner.state.candidates[0].empty);
16370        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16371        assert!(
16372            crate::run::read_artifact(&runner.state, "cand-A.patch")
16373                .unwrap()
16374                .contains("failed seat edits")
16375        );
16376        assert!(wt.exists());
16377    }
16378
16379    #[tokio::test]
16380    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16381        let (_tmp, mut runner) = preservation_resume_fixture().await;
16382        let repo = runner.state.repo.clone();
16383        let common = git::git(
16384            &repo,
16385            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16386        )
16387        .await
16388        .unwrap();
16389        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16390        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16391        assert!(runner.fold_losers().await.is_err());
16392        assert!(!runner.state.candidates[0].worktree.exists());
16393        assert!(!runner.state.candidates[0].folded);
16394        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16395        runner.state = RunState::load(&runner.state.id).unwrap();
16396        std::fs::remove_file(branch_lock).unwrap();
16397        let judge = runner.state.worktree_root().join("judge-1");
16398        git::worktree_add_detached(&repo, &judge, "main")
16399            .await
16400            .unwrap();
16401        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16402        git::git(&repo, &["worktree", "lock", &git::path_arg(&judge)])
16403            .await
16404            .unwrap();
16405        runner.roles.judges = vec![spec("judge")];
16406        assert!(runner.fold_losers().await.is_err());
16407        runner.state = RunState::load(&runner.state.id).unwrap();
16408        assert!(
16409            runner.state.candidates[0].folded,
16410            "candidate progress saved before judge failure"
16411        );
16412        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16413        assert!(judge.join("uncommitted").exists());
16414        git::git(&repo, &["worktree", "unlock", &git::path_arg(&judge)])
16415            .await
16416            .unwrap();
16417        runner.fold_losers().await.unwrap();
16418        assert!(!judge.exists());
16419        runner.fold_losers().await.unwrap();
16420    }
16421
16422    #[tokio::test]
16423    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16424        let (_tmp, mut runner) = preservation_resume_fixture().await;
16425        runner.state.candidates.retain(|c| c.label == 'B');
16426        runner.state.tally = None;
16427        runner.state.judge_skipped = false;
16428        runner.state.prep_complete = Some(true);
16429        runner.state.config.graph.advise = false;
16430        runner.state.config.graph.selected_implementer = Some("alpha".into());
16431        let wt = runner.state.candidates[0].worktree.clone();
16432        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16433        runner
16434            .state
16435            .candidate_preservation
16436            .insert('B', "awaiting persistence".into());
16437        let index_lock = git::git(
16438            &wt,
16439            &[
16440                "rev-parse",
16441                "--path-format=absolute",
16442                "--git-path",
16443                "index.lock",
16444            ],
16445        )
16446        .await
16447        .unwrap();
16448        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16449        runner.state.save().unwrap();
16450        assert!(runner.execute_graph().await.is_err());
16451        assert_eq!(runner.state.status, RunStatus::Blocked);
16452        assert!(runner.state.candidate_preservation.contains_key(&'B'));
16453        std::fs::remove_file(index_lock).unwrap();
16454        // Park at the next graph boundary: this tests the real selected-seat
16455        // hold guard without dispatching reviewers or publishing anything.
16456        for _ in 0..2 {
16457            runner.state = RunState::load(&runner.state.id).unwrap();
16458            runner.pause.park();
16459            runner.execute_graph().await.unwrap();
16460            assert_eq!(runner.state.status, RunStatus::Implementing);
16461            assert!(runner.state.candidate_preservation.is_empty());
16462            assert_eq!(runner.state.candidates[0].commits, 2);
16463            assert!(runner.state.tally.is_none());
16464        }
16465        runner.implement().await.unwrap();
16466        assert_eq!(runner.state.status, RunStatus::Judging);
16467        // A genuine selected-implementer hold must still require release.
16468        runner.state.status = RunStatus::Blocked;
16469        runner.state.candidates[0].failed = Some("implementation failed".into());
16470        runner.state.save().unwrap();
16471        let error = runner.execute_graph().await.unwrap_err();
16472        assert!(
16473            error
16474                .to_string()
16475                .contains("selected implementation is held")
16476        );
16477    }
16478
16479    #[tokio::test]
16480    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16481        let (_tmp, mut runner) = preservation_resume_fixture().await;
16482        let repo = runner.state.repo.clone();
16483        let candidate = runner.state.candidates[0].clone();
16484        // Remove only the disposable fixture directory, leaving Git's linked
16485        // worktree registration and branch intact, as in a vanished checkout.
16486        let registered = git::worktree_holding(&repo, &candidate.branch)
16487            .await
16488            .unwrap()
16489            .unwrap();
16490        assert_eq!(
16491            registered.canonicalize().unwrap(),
16492            candidate.worktree.canonicalize().unwrap()
16493        );
16494        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16495        assert_eq!(
16496            git::worktree_holding(&repo, &candidate.branch)
16497                .await
16498                .unwrap(),
16499            Some(registered)
16500        );
16501        assert!(
16502            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16503            "the stale registration initially keeps the branch checked out"
16504        );
16505        runner.fold_losers().await.unwrap();
16506        assert!(runner.state.candidates[0].folded);
16507        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16508        assert_eq!(
16509            git::worktree_holding(&repo, &candidate.branch)
16510                .await
16511                .unwrap(),
16512            None
16513        );
16514        runner.state = RunState::load(&runner.state.id).unwrap();
16515        runner.fold_losers().await.unwrap();
16516    }
16517
16518    #[tokio::test]
16519    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16520        let (_tmp, mut runner) = preservation_resume_fixture().await;
16521        let repo = runner.state.repo.clone();
16522        let candidate = runner.state.candidates[0].clone();
16523        let winner = runner.state.candidates[1].clone();
16524        let registered = git::worktree_holding(&repo, &candidate.branch)
16525            .await
16526            .unwrap()
16527            .unwrap();
16528        assert_eq!(
16529            registered.canonicalize().unwrap(),
16530            candidate.worktree.canonicalize().unwrap()
16531        );
16532        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16533        assert_eq!(
16534            git::worktree_holding(&repo, &candidate.branch)
16535                .await
16536                .unwrap(),
16537            Some(registered)
16538        );
16539        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16540
16541        for _ in 0..2 {
16542            fold_run(&mut runner.state, false, &crate::run::home())
16543                .await
16544                .unwrap();
16545            runner.state = RunState::load(&runner.state.id).unwrap();
16546            assert!(runner.state.candidates[0].folded);
16547            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16548            assert_eq!(
16549                git::worktree_holding(&repo, &candidate.branch)
16550                    .await
16551                    .unwrap(),
16552                None
16553            );
16554            assert!(!runner.state.candidates[1].folded);
16555            assert!(winner.worktree.exists());
16556            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16557        }
16558    }
16559    #[tokio::test]
16560    async fn fold_recovery_override_archives_sources_and_keeps_safety_checks() {
16561        for sidecar in [false, true] {
16562            let tmp = tempfile::tempdir().unwrap();
16563            let home = tmp.path().join("home");
16564            let repo = tmp.path().join("repo");
16565            std::fs::create_dir(&repo).unwrap();
16566            init_repo(&repo);
16567            let mut state = runner_at(RunStatus::Superseded).state;
16568            state.repo = repo.clone();
16569            state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16570            state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16571            state.driver_exited = true;
16572            state.reserve_under(&home).unwrap();
16573            git::git(
16574                &repo,
16575                &["update-ref", "refs/remotes/origin/main", &state.base_commit],
16576            )
16577            .await
16578            .unwrap();
16579            let wt = state.worktree_root().join("cand-A");
16580            git::worktree_add_branch(&repo, &wt, "candidate", "main")
16581                .await
16582                .unwrap();
16583            std::fs::write(wt.join("independent"), "retained implementation\n").unwrap();
16584            git::commit_all(&wt, "fixture: independent source")
16585                .await
16586                .unwrap();
16587            let head = git::rev_parse(&wt, "HEAD").await.unwrap();
16588            state.candidates.push(Candidate {
16589                index: 0,
16590                label: 'A',
16591                agent: "fixture".into(),
16592                branch: "candidate".into(),
16593                worktree: wt.clone(),
16594                summary: String::new(),
16595                stat: String::new(),
16596                files: 0,
16597                commits: 0,
16598                empty: false,
16599                failed: None,
16600                verified_noop: None,
16601                duration_ms: 0,
16602                folded: false,
16603            });
16604            let mut task = crate::queue::Task::new(
16605                "task".into(),
16606                "task".into(),
16607                repo.clone(),
16608                crate::queue::Source::Human,
16609            );
16610            task.runs.push(state.id.clone());
16611            crate::queue::Queue::at(home.join("queue"))
16612                .put(&mut task)
16613                .unwrap();
16614            if sidecar {
16615                crate::pr_recovery::retain_sources(&task, &home).unwrap();
16616            }
16617            state.save_under(&home).unwrap();
16618            let reason = crate::pr_recovery::source_retention_reason(&home, &state).unwrap();
16619            assert!(
16620                fold_run(&mut state, true, &home)
16621                    .await
16622                    .unwrap_err()
16623                    .to_string()
16624                    .contains(&reason)
16625            );
16626            let mut disk = state.config.disk.clone();
16627            disk.fold_grace_secs = 0;
16628            assert_eq!(
16629                crate::clean::fold_due(
16630                    &home.join("runs"),
16631                    &home,
16632                    &state.worktree_root(),
16633                    &disk,
16634                    Timestamp::now() + std::time::Duration::from_secs(10),
16635                )
16636                .await
16637                .unwrap(),
16638                (0, 0)
16639            );
16640            assert!(wt.exists());
16641            state.status = RunStatus::Blocked;
16642            state.save_under(&home).unwrap();
16643            assert!(
16644                fold_run_overriding_recovery_retention(&mut state, true, &home)
16645                    .await
16646                    .is_err()
16647            );
16648            state.status = RunStatus::Superseded;
16649            state.driver_exited = false;
16650            state.driver_claimed = None;
16651            state.save_under(&home).unwrap();
16652            assert!(
16653                fold_run_overriding_recovery_retention(&mut state, true, &home)
16654                    .await
16655                    .is_err()
16656            );
16657            state.driver_pid = Some(std::process::id());
16658            state.driver_started_at = crate::proc::process_started_at(std::process::id());
16659            state.save_under(&home).unwrap();
16660            assert!(
16661                fold_run_overriding_recovery_retention(&mut state, true, &home)
16662                    .await
16663                    .is_err()
16664            );
16665            state.driver_exited = true;
16666            state.driver_pid = None;
16667            state.driver_started_at = None;
16668            state.save_under(&home).unwrap();
16669            let lease = crate::driver::maintenance(&home, &state.id).unwrap();
16670            assert!(
16671                fold_run_overriding_recovery_retention(&mut state, true, &home)
16672                    .await
16673                    .is_err()
16674            );
16675            drop(lease);
16676            #[cfg(unix)]
16677            {
16678                let output = crate::retention::OutputLease::take(&wt, false).unwrap();
16679                assert!(
16680                    fold_run_overriding_recovery_retention(&mut state, true, &home)
16681                        .await
16682                        .is_err()
16683                );
16684                drop(output);
16685            }
16686            let owner = crate::pr_head::owner_lock(&home, &state.id).unwrap();
16687            assert!(
16688                fold_run_overriding_recovery_retention(&mut state, true, &home)
16689                    .await
16690                    .is_err()
16691            );
16692            drop(owner);
16693            let checkpoint = git::git(
16694                &wt,
16695                &[
16696                    "rev-parse",
16697                    "--path-format=absolute",
16698                    "--git-path",
16699                    "CHERRY_PICK_HEAD",
16700                ],
16701            )
16702            .await
16703            .unwrap();
16704            std::fs::write(&checkpoint, &head).unwrap();
16705            assert!(
16706                fold_run_overriding_recovery_retention(&mut state, true, &home)
16707                    .await
16708                    .is_err()
16709            );
16710            std::fs::remove_file(checkpoint).unwrap();
16711            std::fs::write(wt.join("dirty"), "keep").unwrap();
16712            assert!(
16713                fold_run_overriding_recovery_retention(&mut state, true, &home)
16714                    .await
16715                    .is_err()
16716            );
16717            assert!(wt.join("dirty").exists());
16718            std::fs::remove_file(wt.join("dirty")).unwrap();
16719            let evidence = home
16720                .join("runs")
16721                .join(&state.id)
16722                .join("artifacts/fold-evidence");
16723            std::fs::create_dir_all(evidence.parent().unwrap()).unwrap();
16724            std::fs::write(&evidence, "blocked archive").unwrap();
16725            assert!(
16726                fold_run_overriding_recovery_retention(&mut state, true, &home)
16727                    .await
16728                    .is_err()
16729            );
16730            assert!(wt.exists());
16731            assert!(
16732                !RunState::load_under(&state.id, &home)
16733                    .unwrap()
16734                    .events
16735                    .iter()
16736                    .any(|event| event.message.contains("--override-recovery-retention"))
16737            );
16738            std::fs::remove_file(&evidence).unwrap();
16739            fold_run_overriding_recovery_retention(&mut state, true, &home)
16740                .await
16741                .unwrap();
16742            assert!(!wt.exists());
16743            assert!(!git::branch_exists(&repo, "candidate").await.unwrap());
16744            assert!(
16745                git::worktree_holding(&repo, "candidate")
16746                    .await
16747                    .unwrap()
16748                    .is_none()
16749            );
16750            assert!(
16751                std::fs::read_to_string(evidence.join(format!("{head}.patch")))
16752                    .unwrap()
16753                    .contains("retained implementation")
16754            );
16755            assert!(evidence.join(format!("{head}.index")).is_file());
16756            let saved = RunState::load_under(&state.id, &home).unwrap();
16757            assert_eq!(saved.status, RunStatus::Superseded);
16758            assert!(saved.candidates[0].folded);
16759            assert!(saved.events.iter().any(|event| {
16760                event.message.contains("--override-recovery-retention")
16761                    && event.message.contains(&state.id)
16762                    && event.message.contains(&reason)
16763            }));
16764            assert_eq!(
16765                crate::pr_recovery::explicit_sources_retained(&home, &state.id),
16766                sidecar
16767            );
16768        }
16769    }
16770
16771    #[tokio::test]
16772    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16773        ask_test_home();
16774        let tmp = tempfile::tempdir().unwrap();
16775        let repo = tmp.path().join("repo");
16776        std::fs::create_dir(&repo).unwrap();
16777        init_repo(&repo);
16778        let mut runner = runner_at(RunStatus::Implementing);
16779        runner.state.repo = repo.clone();
16780        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16781        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16782        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16783            let branch = format!("candidate-{label}");
16784            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16785            git::worktree_add_branch(&repo, &wt, &branch, "main")
16786                .await
16787                .unwrap();
16788            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16789            git::git(&wt, &["add", "README.md"]).await.unwrap();
16790            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16791            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16792            runner.state.candidates.push(Candidate {
16793                index,
16794                label,
16795                agent: "alpha".to_owned(),
16796                branch,
16797                worktree: wt,
16798                summary: "completed implementation".to_owned(),
16799                stat: String::new(),
16800                files: 0,
16801                commits: 0,
16802                empty: false,
16803                failed: None,
16804                verified_noop: None,
16805                duration_ms: 42,
16806                folded: false,
16807            });
16808            runner
16809                .state
16810                .candidate_preservation
16811                .insert(label, "awaiting persistence".to_owned());
16812        }
16813        let wt = runner.state.candidates[0].worktree.clone();
16814        let index_lock = git::git(
16815            &wt,
16816            &[
16817                "rev-parse",
16818                "--path-format=absolute",
16819                "--git-path",
16820                "index.lock",
16821            ],
16822        )
16823        .await
16824        .unwrap();
16825        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16826        runner.state.save().unwrap();
16827        assert!(runner.persist_candidates().await.is_err());
16828        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16829        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16830        assert_eq!(
16831            runner.state.candidates[1].commits, 1,
16832            "other results are still collected"
16833        );
16834        assert!(
16835            !runner.state.candidates[0].empty,
16836            "failed rescue is not no change"
16837        );
16838        assert_eq!(
16839            runner.state.candidates[0].summary,
16840            "completed implementation"
16841        );
16842        assert!(runner.after_implement().is_err());
16843        let home = tmp.path().join("fold-home");
16844        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16845        assert!(wt.join("new-file").exists());
16846        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16847        assert!(recovery.join("staged.patch").exists());
16848        assert!(recovery.join("unstaged.patch").exists());
16849        assert_eq!(
16850            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16851            "untracked edits\n"
16852        );
16853        runner.state.begin_pass();
16854        runner.state.end_pass_parked(ParkCause::Upgrade {
16855            from: "1.0".into(),
16856            to: Some("1.1".into()),
16857        });
16858        runner.state.save().unwrap();
16859        runner.state = RunState::load(&runner.state.id).unwrap();
16860        assert!(matches!(
16861            runner.state.passes.last().unwrap().end,
16862            Some(crate::run::PassEnd::Parked {
16863                cause: ParkCause::Upgrade { .. }
16864            })
16865        ));
16866        std::fs::remove_file(index_lock).unwrap();
16867        // No implementer is configured: a persistence retry cannot dispatch a seat.
16868        runner.implement().await.unwrap();
16869        assert!(runner.state.candidate_preservation.is_empty());
16870        assert!(
16871            runner
16872                .state
16873                .candidates
16874                .iter()
16875                .all(|c| c.commits == 1 && !c.empty)
16876        );
16877        assert!(git::is_clean(&wt).await.unwrap());
16878        assert!(
16879            crate::run::read_artifact(&runner.state, "cand-A.patch")
16880                .unwrap()
16881                .contains("later tracked edits")
16882        );
16883    }
16884
16885    #[tokio::test]
16886    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16887        ask_test_home();
16888        let tmp = tempfile::tempdir().unwrap();
16889        let repo = tmp.path().join("repo");
16890        std::fs::create_dir(&repo).unwrap();
16891        init_repo(&repo);
16892        let wt = tmp.path().join("candidate");
16893        git::worktree_add_branch(&repo, &wt, "candidate", "main")
16894            .await
16895            .unwrap();
16896        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16897        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16898        assert!(wt.join("uncommitted").exists());
16899        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16900        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16901        let mut runner = runner_at(RunStatus::Implementing);
16902        runner.state.repo = repo.clone();
16903        runner.state.base_commit = "missing-base".to_owned();
16904        runner.state.candidates.push(Candidate {
16905            index: 0,
16906            label: 'A',
16907            agent: "alpha".into(),
16908            branch: "candidate".into(),
16909            worktree: wt.clone(),
16910            summary: "completed".into(),
16911            stat: String::new(),
16912            files: 0,
16913            commits: 0,
16914            empty: false,
16915            failed: None,
16916            verified_noop: None,
16917            duration_ms: 1,
16918            folded: false,
16919        });
16920        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16921        assert!(
16922            fold_run(&mut runner.state, true, &tmp.path().join("home"))
16923                .await
16924                .is_err()
16925        );
16926        assert!(!runner.state.candidates[0].folded);
16927        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16928        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16929        runner
16930            .state
16931            .candidate_preservation
16932            .insert('A', "pending".into());
16933        assert!(runner.persist_candidates().await.is_err());
16934        assert!(!runner.state.candidates[0].empty);
16935        assert!(!runner.state.candidates[0].folded);
16936        assert!(wt.exists());
16937        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16938        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16939        std::fs::create_dir_all(&patch_path).unwrap();
16940        assert!(
16941            runner.persist_candidates().await.is_err(),
16942            "artifact failure stays pending"
16943        );
16944        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16945        assert!(!runner.state.candidates[0].empty);
16946        std::fs::remove_dir(patch_path).unwrap();
16947        runner.persist_candidates().await.unwrap();
16948        assert!(
16949            runner.state.candidates[0].empty,
16950            "only successful inspection confirms no change"
16951        );
16952    }
16953
16954    fn init_repo(dir: &Path) {
16955        let run = |args: &[&str]| {
16956            let out = std::process::Command::new("git")
16957                .args(args)
16958                .current_dir(dir)
16959                .quiet()
16960                .output()
16961                .expect("spawn git");
16962            assert!(
16963                out.status.success(),
16964                "git {args:?} failed: {}",
16965                String::from_utf8_lossy(&out.stderr)
16966            );
16967        };
16968        run(&["init", "-b", "main"]);
16969        run(&["config", "user.name", "magi test"]);
16970        run(&["config", "user.email", "magi@example.com"]);
16971        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16972        run(&["add", "-A"]);
16973        run(&["commit", "-m", "init"]);
16974    }
16975
16976    // `settle_questions` is what closes the ghost the phone showed: a run's
16977    // seat asked something, the run then ended, and nothing was left to
16978    // abandon the question it left `open`. `HOME` is a process-wide
16979    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
16980    // first time it runs in the binary — every test below still reaches the
16981    // same directory whichever call won, and each gets its own run id from
16982    // `RunState::new`, so they never collide there.
16983    fn ask_test_home() {
16984        crate::run::pin_test_home();
16985    }
16986
16987    #[test]
16988    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16989        ask_test_home();
16990        let mut runner = runner_at(RunStatus::Prep);
16991        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16992        // A blocked temporary output path makes the terminal save fail while
16993        // preserving the already published reservation.
16994        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16995        let error = guard
16996            .finish(
16997                &mut runner.state,
16998                Err(
16999                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
17000                        .into(),
17001                ),
17002            )
17003            .unwrap_err();
17004        assert_eq!(
17005            error.downcast_ref::<std::io::Error>().unwrap().kind(),
17006            std::io::ErrorKind::PermissionDenied
17007        );
17008        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
17009        let message = format!("{error:#}");
17010        assert!(message.contains("original refusal"));
17011        assert!(!message.contains("could not save terminal"));
17012        assert_eq!(
17013            error.chain().count(),
17014            2,
17015            "only startup attribution wraps the original error"
17016        );
17017        let saved = RunState::load(&runner.state.id).unwrap();
17018        assert_eq!(saved.status, RunStatus::Blocked);
17019        assert!(!saved.startup.unwrap().complete);
17020    }
17021
17022    #[test]
17023    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
17024        ask_test_home();
17025        let mut runner = runner_at(RunStatus::Prep);
17026        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17027        let id = runner.state.id.clone();
17028        drop(guard);
17029        let saved = RunState::load(&id).unwrap();
17030        assert_eq!(saved.status, RunStatus::Blocked);
17031        assert!(!saved.startup.as_ref().unwrap().complete);
17032        assert!(
17033            saved
17034                .startup
17035                .unwrap()
17036                .failure
17037                .unwrap()
17038                .contains("interrupted")
17039        );
17040        assert!(Runner::resume(&id).is_err());
17041    }
17042
17043    #[test]
17044    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
17045        ask_test_home();
17046        let mut runner = runner_at(RunStatus::Prep);
17047        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17048        let error = guard
17049            .finish(
17050                &mut runner.state,
17051                Err(
17052                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
17053                        .into(),
17054                ),
17055            )
17056            .unwrap_err();
17057        assert_eq!(
17058            error.downcast_ref::<std::io::Error>().unwrap().kind(),
17059            std::io::ErrorKind::PermissionDenied
17060        );
17061        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
17062        let mut task = crate::queue::Task::new(
17063            "review".into(),
17064            "criteria".into(),
17065            runner.state.repo.clone(),
17066            crate::queue::Source::Human,
17067        );
17068        task.record_startup_failure(failure.clone());
17069        task.record_startup_failure(failure.clone());
17070        assert_eq!(task.startup_failures.len(), 1);
17071        assert!(task.runs.is_empty());
17072        assert_eq!(task.attempts, 0);
17073    }
17074
17075    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
17076        ask_test_home();
17077        let tmp = tempfile::tempdir().unwrap();
17078        let repo = tmp.path().join("repo");
17079        std::fs::create_dir(&repo).unwrap();
17080        init_repo(&repo);
17081        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17082        let origin = tmp.path().join("origin.git");
17083        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
17084            .await
17085            .unwrap();
17086        git::git(
17087            &repo,
17088            &["remote", "add", "origin", origin.to_str().unwrap()],
17089        )
17090        .await
17091        .unwrap();
17092        git::git(&repo, &["checkout", "-b", "fix/retained"])
17093            .await
17094            .unwrap();
17095        std::fs::write(repo.join("retained"), "retained change").unwrap();
17096        git::git(&repo, &["add", "."]).await.unwrap();
17097        git::git(&repo, &["commit", "-m", "retained implementation"])
17098            .await
17099            .unwrap();
17100        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
17101        git::git(&repo, &["checkout", "main"]).await.unwrap();
17102        std::fs::write(repo.join("base-change"), "new base").unwrap();
17103        git::git(&repo, &["add", "."]).await.unwrap();
17104        git::git(&repo, &["commit", "-m", "base moved"])
17105            .await
17106            .unwrap();
17107        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17108        git::git(&repo, &["checkout", "fix/retained"])
17109            .await
17110            .unwrap();
17111        let mut runner = runner_at(RunStatus::Voting);
17112        runner.state.repo = repo.clone();
17113        runner.state.base_commit = base;
17114        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17115        runner.state.config.disk.min_free_bytes = 0;
17116        runner.state.config.graph.review_rounds = 1;
17117        runner.state.config.verify.e2e = vec!["true".into()];
17118        runner.state.config.verify.gate = vec!["true".into()];
17119        runner.state.candidates = vec![Candidate {
17120            index: 0,
17121            label: 'A',
17122            agent: "fixture".into(),
17123            branch: "fix/retained".into(),
17124            worktree: repo.clone(),
17125            summary: String::new(),
17126            stat: String::new(),
17127            files: 1,
17128            commits: 1,
17129            empty: false,
17130            failed: None,
17131            verified_noop: None,
17132            duration_ms: 0,
17133            folded: false,
17134        }];
17135        runner.state.tally = Some(Tally {
17136            first_choice: BTreeMap::new(),
17137            borda: BTreeMap::new(),
17138            winner: 'A',
17139            rankings: 0,
17140            unanimous_initial: true,
17141            deliberated: false,
17142            changed_votes: 0,
17143            unanimous_final: true,
17144            tie_break: None,
17145            judges: 0,
17146            present: 0,
17147            quorum: 0,
17148            met_quorum: true,
17149            uncontested: Some("review-only".into()),
17150        });
17151        let mut task = crate::queue::Task::new(
17152            "fixture".into(),
17153            "fixture".into(),
17154            repo,
17155            crate::queue::Source::Human,
17156        );
17157        task.link_run(&runner.state.id);
17158        crate::queue::Queue::open().put(&mut task).unwrap();
17159        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
17160        runner.state.base_sync = Some(BaseSync {
17161            tip: String::new(),
17162            behind: 1,
17163            attempts: BASE_SYNC_ROUNDS,
17164            conflict: None,
17165            already_in: None,
17166        });
17167        runner.sync_to_base().await.unwrap();
17168        assert_eq!(runner.state.status, RunStatus::Blocked);
17169        (tmp, runner, head, task.id)
17170    }
17171
17172    #[tokio::test]
17173    async fn rebase_preparation_failure_preserves_base_sync_budget_and_resumes() {
17174        let (_tmp, mut runner, head, _task) = exhausted_base_fixture().await;
17175        runner.state.status = RunStatus::Voting;
17176        runner.state.base_sync = None;
17177        runner.state.base_sync_recovery = Default::default();
17178        let scratch = runner.state.dir().join("base-sync");
17179        std::fs::create_dir_all(&scratch).unwrap();
17180        std::fs::write(scratch.join("obstruction"), "keep").unwrap();
17181        runner.sync_to_base().await.unwrap();
17182        assert!(runner.state.resource_block.is_some());
17183        assert!(!runner.state.base_sync_blocked());
17184        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 0);
17185        assert_eq!(runner.state.base_episode_used(), 0);
17186        assert!(runner.state.base_sync_recovery.operations.is_empty());
17187        assert!(runner.state.rebase_fixes.is_empty());
17188        assert_eq!(
17189            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17190            head
17191        );
17192        std::fs::remove_dir_all(&scratch).unwrap();
17193        runner.state.resource_block = None;
17194        runner.sync_to_base().await.unwrap();
17195        assert!(!runner.state.base_sync_blocked());
17196        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17197        assert_eq!(runner.state.base_episode_used(), 1);
17198        assert!(runner.state.base_sync_recovery.operations[0].finished);
17199        assert!(runner.state.rebase_fixes.is_empty());
17200        assert!(git::is_ancestor(&runner.state.repo, "origin/main", "HEAD").await);
17201        assert!(git::is_clean(&runner.state.repo).await.unwrap());
17202    }
17203
17204    #[cfg(unix)]
17205    #[tokio::test]
17206    async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
17207        for routed in [false, true] {
17208            let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
17209            let home = crate::run::home();
17210            let queue = crate::queue::Queue::open();
17211            let task = queue.get(&owner).unwrap();
17212            runner.state.task_instruction = Some(task.instruction.clone());
17213            runner.state.driver_exited = true;
17214            runner.state.save().unwrap();
17215            let questions = crate::ask::Questions::at(home.join("questions"));
17216            let mut q = crate::ask::Question::new(
17217                runner.state.id.clone(),
17218                "base-recovery".into(),
17219                "owner".into(),
17220                "Recover bounded base movement?".into(),
17221                String::new(),
17222                vec!["recover".into()],
17223            );
17224            q.actions.insert(
17225                "recover".into(),
17226                crate::ask::ChoiceAction::RecoverBase {
17227                    run: runner.state.id.clone(),
17228                    stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
17229                    episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
17230                },
17231            );
17232            q.routed_task = Some(owner.clone());
17233            q.answer(crate::ask::Answer::Choice("recover".into()))
17234                .unwrap();
17235            questions.put(&mut q).unwrap();
17236            crate::base_recovery_control::recover(
17237                &queue,
17238                &home,
17239                crate::base_recovery_control::Request {
17240                    run: runner.state.id.clone(),
17241                    head: head.clone(),
17242                    actor: "decisions".into(),
17243                    request: q.id.clone(),
17244                    decision: Some((owner.clone(), q.answered_at.unwrap())),
17245                    expected_stop: Some((
17246                        runner.state.base_sync_recovery.stop.clone().unwrap(),
17247                        runner.state.base_sync_recovery.episodes.last().unwrap().id,
17248                    )),
17249                },
17250            )
17251            .await
17252            .unwrap();
17253            if routed {
17254                questions
17255                    .update(&q.id, |q| {
17256                        q.routed_task = Some("another-owner".into());
17257                        Ok(())
17258                    })
17259                    .unwrap();
17260            } else {
17261                let mut task = queue.get(&owner).unwrap();
17262                task.hold_manual(Some("new owner decision".into()));
17263                // Keep the generation to exercise the separate owner-decision binding.
17264                task.status = crate::queue::TaskStatus::Queued;
17265                queue.put(&mut task).unwrap();
17266            }
17267            runner.state = RunState::load(&runner.state.id).unwrap();
17268            runner.sync_to_base().await.unwrap();
17269            let approval = runner
17270                .state
17271                .base_sync_recovery
17272                .authorizations
17273                .last()
17274                .unwrap();
17275            assert!(approval.consumed);
17276            assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
17277                "routing"
17278            } else {
17279                "owner decision"
17280            }));
17281            assert_eq!(
17282                runner.state.base_sync.as_ref().unwrap().attempts,
17283                BASE_SYNC_ROUNDS
17284            );
17285            assert_eq!(
17286                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17287                head
17288            );
17289        }
17290    }
17291
17292    #[cfg(unix)]
17293    #[tokio::test]
17294    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
17295        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
17296        if std::env::var_os(CHILD).is_none() {
17297            let home = tempfile::tempdir().unwrap();
17298            let output = std::process::Command::new(std::env::current_exe().unwrap())
17299                .args([
17300                    "--exact",
17301                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
17302                    "--nocapture",
17303                ])
17304                .env(CHILD, "1")
17305                .env("MAGI_HOME", home.path())
17306                .env("MAGI_WORKER_HOME", home.path())
17307                .env("MAGI_WORKER_ID", "abcdef")
17308                .output()
17309                .unwrap();
17310            assert!(
17311                output.status.success()
17312                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17313                "{}\n{}",
17314                String::from_utf8_lossy(&output.stdout),
17315                String::from_utf8_lossy(&output.stderr)
17316            );
17317            return;
17318        }
17319        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
17320        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17321        let home = crate::run::home();
17322        let queue = crate::queue::Queue::open();
17323        let mut owner = queue.get(&task).unwrap();
17324        owner.release_operator("test-release", None);
17325        owner.start(runner.state.id.clone());
17326        queue.put(&mut owner).unwrap();
17327        runner.state.driver_exited = true;
17328        runner.state.task_instruction = Some(owner.instruction.clone());
17329        runner.state.save().unwrap();
17330        let _claim = queue.claim(&task).unwrap();
17331        let reserve = |id: &str| {
17332            let directory = home.join("workers/jobs").join(id);
17333            std::fs::create_dir_all(&directory).unwrap();
17334            let job = crate::worker::Job {
17335                id: id.into(),
17336                sequence: 1,
17337                kind: "task".into(),
17338                key: format!("task:{task}"),
17339                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
17340                generation: "fixture-generation".into(),
17341                version: None,
17342                status: crate::worker::Status::Running,
17343                result: None,
17344                pid: Some(std::process::id()),
17345                created_at: 0,
17346                finished_at: None,
17347            };
17348            std::fs::write(
17349                directory.join("job.json"),
17350                serde_json::to_vec(&job).unwrap(),
17351            )
17352            .unwrap();
17353            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
17354        };
17355        let _own = reserve("abcdef");
17356        let other = reserve("fedcba");
17357        assert!(
17358            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17359                .await
17360                .is_err()
17361        );
17362        assert!(
17363            RunState::load(&runner.state.id)
17364                .unwrap()
17365                .base_sync_recovery
17366                .authorizations
17367                .is_empty()
17368        );
17369        drop(other);
17370        assert!(crate::daemon::is_working_on(
17371            &home,
17372            &runner.state.id,
17373            Timestamp::now()
17374        ));
17375        assert!(
17376            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
17377        );
17378        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17379            .await
17380            .unwrap();
17381        assert!(
17382            RunState::load(&runner.state.id)
17383                .unwrap()
17384                .base_recovery_pending()
17385        );
17386    }
17387
17388    #[cfg(unix)]
17389    #[tokio::test]
17390    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
17391        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17392        let home = crate::run::home();
17393        runner.state.driver_pid = Some(std::process::id());
17394        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
17395        runner.state.driver_exited = false;
17396        runner.state.begin_pass();
17397        runner.state.save().unwrap();
17398        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
17399        driver.record(&runner.state).unwrap();
17400        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
17401        // Preserve the durable claim as if the process died before Drop saved.
17402        drop(driver);
17403        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
17404        assert_eq!(
17405            runner.state.liveness_under(&home, false),
17406            crate::run::Liveness::Dead
17407        );
17408        let queue = crate::queue::Queue::open();
17409        let _claim = queue.claim(&task).unwrap();
17410        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17411            .await
17412            .unwrap();
17413        let disk = RunState::load(&runner.state.id).unwrap();
17414        assert!(!disk.driver_exited);
17415        assert!(disk.base_recovery_pending());
17416    }
17417
17418    #[cfg(unix)]
17419    #[tokio::test]
17420    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
17421        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
17422        if std::env::var_os(CHILD).is_none() {
17423            use std::os::unix::fs::PermissionsExt;
17424            let tmp = tempfile::tempdir().unwrap();
17425            let gh = tmp.path().join("gh");
17426            std::fs::write(&gh, r#"#!/bin/sh
17427case "$1 $2" in
17428  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
17429  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
17430  *) exit 2 ;;
17431esac
17432"#).unwrap();
17433            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
17434            let mut paths = vec![tmp.path().to_path_buf()];
17435            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
17436            let output = std::process::Command::new(std::env::current_exe().unwrap())
17437                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
17438                .env(CHILD, "1")
17439                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
17440                .env("PATH", std::env::join_paths(paths).unwrap())
17441                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17442                .output().unwrap();
17443            assert!(
17444                output.status.success()
17445                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17446                "{}\n{}",
17447                String::from_utf8_lossy(&output.stdout),
17448                String::from_utf8_lossy(&output.stderr)
17449            );
17450            return;
17451        }
17452        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17453        let repo = runner.state.repo.clone();
17454        git::git(&repo, &["checkout", "main"]).await.unwrap();
17455        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17456        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17457        git::git(&repo, &["checkout", "fix/retained"])
17458            .await
17459            .unwrap();
17460        runner.state.pr = Some(crate::run::PrRecord {
17461            url: "https://example.test/project/pull/1".into(),
17462            number: 1,
17463            state: "open".into(),
17464            checks: "unknown".into(),
17465            round: 0,
17466            rounds: 1,
17467            red_at_merge: Vec::new(),
17468        });
17469        runner.sync_to_base().await.unwrap();
17470        assert_eq!(runner.state.status, RunStatus::Blocked);
17471        assert_eq!(
17472            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17473            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17474        );
17475        assert!(runner.state.base_sync_closure_pending());
17476        assert!(crate::handover::can_resume(&runner.state));
17477        let queue = crate::queue::Queue::open();
17478        assert!(!crate::base_sync_recovery::held_task(
17479            &queue,
17480            &queue.get(&task).unwrap()
17481        ));
17482        assert!(runner.state.authorize_base_sync(&task).is_err());
17483        // Legacy closure stops have no typed stop but must retain the checkpoint.
17484        runner.state.base_sync_recovery.stop = None;
17485        runner.state.status = RunStatus::Voting;
17486        runner.state.gate_ran = true;
17487        runner.state.config.graph.review_rounds = 0;
17488        runner.state.save().unwrap();
17489        assert!(crate::handover::can_resume(&runner.state));
17490        runner.state = RunState::load(&runner.state.id).unwrap();
17491        runner.execute_graph().await.unwrap();
17492        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17493        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17494        assert_eq!(
17495            runner.state.base_sync.as_ref().unwrap().attempts,
17496            BASE_SYNC_ROUNDS
17497        );
17498        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17499        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17500        assert!(runner.state.reviews.is_empty());
17501        assert!(runner.state.gate.is_empty());
17502    }
17503
17504    #[cfg(unix)]
17505    #[tokio::test]
17506    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17507        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17508        runner.state.driver_exited = true;
17509        runner.state.save().unwrap();
17510        let home = crate::run::home();
17511        let queue = crate::queue::Queue::open();
17512        let _claim = queue.claim(&task).unwrap();
17513        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17514        assert!(
17515            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17516                .await
17517                .is_err()
17518        );
17519        drop(lease);
17520        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17521        assert!(
17522            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17523                .await
17524                .is_err()
17525        );
17526        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17527        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17528            .await
17529            .unwrap();
17530        let disk = RunState::load(&runner.state.id).unwrap();
17531        assert!(disk.base_recovery_pending());
17532        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17533        assert!(
17534            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17535                .await
17536                .is_err()
17537        );
17538        assert_eq!(
17539            RunState::load(&runner.state.id)
17540                .unwrap()
17541                .base_sync_recovery
17542                .authorizations
17543                .len(),
17544            1
17545        );
17546        let mut owner = queue.get(&task).unwrap();
17547        owner.runs.push("20260101-000000-ffff".into());
17548        queue.put(&mut owner).unwrap();
17549        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17550            .await
17551            .unwrap_err();
17552        assert!(error.to_string().contains("run task ownership changed"));
17553    }
17554
17555    #[cfg(unix)]
17556    #[tokio::test]
17557    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17558        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17559        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17560        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17561        legacy["status"] = serde_json::json!("voting");
17562        runner.state = serde_json::from_value(legacy).unwrap();
17563        runner.state.save().unwrap();
17564        for _ in 0..3 {
17565            runner.execute_graph().await.unwrap();
17566            assert_eq!(runner.state.status, RunStatus::Blocked);
17567            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17568            assert!(!crate::handover::can_resume(&runner.state));
17569            assert_eq!(
17570                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17571                head
17572            );
17573            assert!(runner.state.reviews.is_empty());
17574            assert!(!runner.state.gate_ran);
17575        }
17576        let notice = crate::notices::run_ended(&runner.state).unwrap();
17577        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17578        runner.state.begin_pass();
17579        runner.state.status = RunStatus::Voting;
17580        runner.deliberate().await.unwrap();
17581        runner.vote().await.unwrap();
17582        runner.finish_after_tally().await.unwrap();
17583        assert_eq!(runner.state.status, RunStatus::Blocked);
17584        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17585        assert_eq!(
17586            crate::notices::run_ended(&runner.state).unwrap().since,
17587            notice.since
17588        );
17589    }
17590
17591    #[cfg(unix)]
17592    #[tokio::test]
17593    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17594        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17595        for detail in [
17596            "fix/retained conflicts with origin/main and did not rebase: conflict",
17597            "rebased fix/retained locally but origin refused the push",
17598            "unknown failure",
17599        ] {
17600            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17601            runner.state.record_base_sync_stop(&head);
17602            assert!(runner.state.authorize_base_sync(&task).is_err());
17603            runner.state.status = RunStatus::Voting;
17604            runner.execute_graph().await.unwrap();
17605            assert_eq!(runner.state.status, RunStatus::Blocked);
17606            assert!(!crate::handover::can_resume(&runner.state));
17607            assert_eq!(
17608                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17609                head
17610            );
17611        }
17612    }
17613
17614    #[cfg(unix)]
17615    #[tokio::test]
17616    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17617        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17618        runner.state.authorize_base_sync(&task).unwrap();
17619        assert!(runner.state.consume_base_recovery(&head));
17620        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17621        runner.state.reserve_base_operation(&head, &tip);
17622        runner.state.save().unwrap();
17623        runner.state = RunState::load(&runner.state.id).unwrap();
17624        assert!(!runner.state.consume_base_recovery(&head));
17625        runner.sync_to_base().await.unwrap();
17626        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17627        assert_eq!(runner.state.base_episode_used(), 1);
17628        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17629        assert!(
17630            runner
17631                .state
17632                .base_sync_recovery
17633                .operations
17634                .last()
17635                .unwrap()
17636                .finished
17637        );
17638        assert!(!runner.state.base_sync_blocked());
17639        runner.sync_to_base().await.unwrap();
17640        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17641    }
17642
17643    #[cfg(unix)]
17644    #[tokio::test]
17645    async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17646        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17647        // Both historical diagnostic spellings retain an exhaustion identity.
17648        let original = runner
17649            .state
17650            .base_sync
17651            .as_ref()
17652            .unwrap()
17653            .conflict
17654            .clone()
17655            .unwrap();
17656        runner.state.base_sync.as_mut().unwrap().conflict = Some(
17657            original
17658                .replace("synchronization(s)", "rebase(s)")
17659                .replace("synchronizing", "rebasing"),
17660        );
17661        runner.state.base_sync_recovery = Default::default();
17662        runner.state.record_base_sync_stop(&head);
17663        assert_eq!(
17664            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17665            crate::base_sync_recovery::StopKind::Exhausted
17666        );
17667        runner.state.authorize_base_sync(&task).unwrap();
17668        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17669        let auth = legacy["base_sync_recovery"]["authorizations"][0]
17670            .as_object_mut()
17671            .unwrap();
17672        auth.remove("interval");
17673        auth.remove("scope");
17674        runner.state = serde_json::from_value(legacy).unwrap();
17675        assert!(runner.state.consume_base_recovery(&head));
17676        assert_eq!(runner.state.base_episode_used(), 3);
17677        runner.sync_to_base().await.unwrap();
17678        assert_eq!(runner.state.base_episode_used(), 4);
17679        runner.state.resource_block = Some(crate::run::ResourceBlock {
17680            node: "gate".into(),
17681            kind: "cache".into(),
17682            detail: "legacy wait".into(),
17683            at: Timestamp::now(),
17684            cause: crate::run::ResourceBlockCause::Unrecorded,
17685        });
17686        runner.state.save().unwrap();
17687        assert!(
17688            runner
17689                .state
17690                .base_sync_recovery
17691                .episodes
17692                .last()
17693                .unwrap()
17694                .boundary
17695                .is_none()
17696        );
17697        runner.state.resource_block = None;
17698        advance_base_fixture(&runner).await;
17699        runner.sync_to_base().await.unwrap();
17700        assert!(runner.state.base_sync_blocked());
17701        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17702    }
17703
17704    #[cfg(unix)]
17705    async fn advance_base_fixture(runner: &Runner) {
17706        let repo = &runner.state.repo;
17707        for args in [
17708            vec!["checkout", "main"],
17709            vec!["commit", "--allow-empty", "-m", "base advanced"],
17710            vec!["push", "origin", "main"],
17711            vec!["checkout", "fix/retained"],
17712        ] {
17713            git::git(repo, &args).await.unwrap();
17714        }
17715    }
17716
17717    #[cfg(unix)]
17718    fn fresh_base_interval_fixture(runner: &mut Runner) {
17719        runner.state.base_sync.as_mut().unwrap().attempts = 0;
17720        runner.state.base_sync.as_mut().unwrap().conflict = None;
17721        runner.state.base_sync_recovery = Default::default();
17722        runner.state.status = RunStatus::Voting;
17723    }
17724
17725    #[cfg(unix)]
17726    #[tokio::test]
17727    async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17728        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17729        fresh_base_interval_fixture(&mut runner);
17730        runner.state.begin_pass();
17731        runner.sync_to_base().await.unwrap();
17732        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17733        runner.state.resource_block = Some(crate::run::ResourceBlock {
17734            node: "gate".into(),
17735            kind: "build cache".into(),
17736            detail: "busy".into(),
17737            at: Timestamp::now(),
17738            cause: crate::run::ResourceBlockCause::Timeout {
17739                waited_secs: 1,
17740                limit_secs: 1,
17741            },
17742        });
17743        runner.state.save().unwrap();
17744        let queue = crate::queue::Queue::open();
17745        let mut owner = queue.get(&task).unwrap();
17746        runner.state.task_instruction = Some(owner.instruction.clone());
17747        owner.release_operator("fixture-owner", None);
17748        queue.put(&mut owner).unwrap();
17749        let approval = owner.resumption.clone().unwrap();
17750        runner.state.resource_block = None;
17751        runner.state.save().unwrap();
17752        runner.state = RunState::load(&runner.state.id).unwrap();
17753        advance_base_fixture(&runner).await;
17754        runner.sync_to_base().await.unwrap();
17755        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17756        assert_eq!(runner.state.base_episode_used(), 1);
17757        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17758        assert!(
17759            runner.state.base_sync_recovery.episodes[0]
17760                .boundary
17761                .as_ref()
17762                .unwrap()
17763                .consumed
17764        );
17765        assert!(
17766            !runner
17767                .state
17768                .renew_base_episode(&task, &approval, &head, "base")
17769        );
17770        for _ in 0..3 {
17771            owner.release_operator("fixture-owner", None);
17772            queue.put(&mut owner).unwrap();
17773            runner.state = RunState::load(&runner.state.id).unwrap();
17774            runner.state.begin_pass();
17775            runner.sync_to_base().await.unwrap();
17776            assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17777            assert_eq!(runner.state.base_episode_used(), 1);
17778        }
17779        runner.state.resource_block = Some(crate::run::ResourceBlock {
17780            node: "gate".into(),
17781            kind: "disk pressure".into(),
17782            detail: "free space required".into(),
17783            at: Timestamp::now(),
17784            cause: crate::run::ResourceBlockCause::Other,
17785        });
17786        runner.state.end_pass_parked(ParkCause::Interrupt {
17787            reason: "disk pressure".into(),
17788        });
17789        runner.state.save().unwrap();
17790        owner.release_operator("fixture-owner-after-disk", None);
17791        queue.put(&mut owner).unwrap();
17792        runner.state.resource_block = None;
17793        advance_base_fixture(&runner).await;
17794        runner.sync_to_base().await.unwrap();
17795        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17796        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17797        assert_eq!(runner.state.base_episode_used(), 1);
17798        assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17799    }
17800
17801    #[cfg(unix)]
17802    #[tokio::test]
17803    async fn base_recovery_continuous_advancement_stops_at_four() {
17804        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17805        fresh_base_interval_fixture(&mut runner);
17806        for spent in 1..=BASE_SYNC_ROUNDS {
17807            runner.sync_to_base().await.unwrap();
17808            assert!(!runner.state.base_sync_blocked());
17809            assert_eq!(runner.state.base_episode_used(), spent);
17810            advance_base_fixture(&runner).await;
17811        }
17812        runner.sync_to_base().await.unwrap();
17813        assert_eq!(
17814            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17815            crate::base_sync_recovery::StopKind::Exhausted
17816        );
17817        let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17818        assert!(stop.unverified_sync);
17819        let action = runner.state.base_sync_next_action();
17820        assert!(action.contains("never reached its gate"));
17821        assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17822        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17823        runner.state = RunState::load(&runner.state.id).unwrap();
17824        runner.sync_to_base().await.unwrap();
17825        assert_eq!(runner.state.base_episode_used(), 4);
17826        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17827    }
17828
17829    #[cfg(unix)]
17830    #[tokio::test]
17831    async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17832        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17833        runner.state.ensure_base_episode();
17834        // Verification progress alone is not evidence of a successful sync.
17835        runner.state.record_base_progress(&head);
17836        runner.state.base_sync_recovery.stop = None;
17837        runner.state.record_base_sync_stop(&head);
17838        assert!(
17839            !runner
17840                .state
17841                .base_sync_recovery
17842                .stop
17843                .as_ref()
17844                .unwrap()
17845                .unverified_sync
17846        );
17847
17848        runner.state.finish_base_operation(Some(&head));
17849        // A gate for another head cannot verify this synchronization.
17850        runner.state.gate_ran = true;
17851        runner.state.gate_head = Some("older-head".into());
17852        runner.state.base_sync_recovery.stop = None;
17853        runner.state.record_base_sync_stop(&head);
17854        runner.state.save().unwrap();
17855        runner.state = RunState::load(&runner.state.id).unwrap();
17856        assert!(
17857            runner
17858                .state
17859                .base_sync_recovery
17860                .stop
17861                .as_ref()
17862                .unwrap()
17863                .unverified_sync
17864        );
17865
17866        runner.state.gate_head = Some(head.clone());
17867        runner.state.base_sync_recovery.stop = None;
17868        runner.state.record_base_sync_stop(&head);
17869        assert!(
17870            !runner
17871                .state
17872                .base_sync_recovery
17873                .stop
17874                .as_ref()
17875                .unwrap()
17876                .unverified_sync
17877        );
17878        assert!(
17879            !runner
17880                .state
17881                .base_sync_next_action()
17882                .contains("never reached its gate")
17883        );
17884
17885        // Historical records cannot establish the missing synchronization evidence.
17886        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17887        legacy["base_sync_recovery"]["stop"]
17888            .as_object_mut()
17889            .unwrap()
17890            .remove("unverified_sync");
17891        legacy["base_sync_recovery"]["episodes"][0]
17892            .as_object_mut()
17893            .unwrap()
17894            .remove("last_sync_head");
17895        runner.state = serde_json::from_value(legacy).unwrap();
17896        assert!(
17897            !runner
17898                .state
17899                .base_sync_recovery
17900                .stop
17901                .as_ref()
17902                .unwrap()
17903                .unverified_sync
17904        );
17905        assert!(
17906            runner.state.base_sync_recovery.episodes[0]
17907                .last_sync_head
17908                .is_none()
17909        );
17910    }
17911
17912    #[cfg(unix)]
17913    #[tokio::test]
17914    async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17915        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17916        runner.state.authorize_base_sync(&task).unwrap();
17917        assert!(
17918            runner
17919                .state
17920                .base_sync_next_action()
17921                .contains("Approval recorded")
17922        );
17923        assert!(
17924            runner
17925                .state
17926                .base_sync_next_action()
17927                .contains("4/4 in episode 0, 4 lifetime")
17928        );
17929        assert!(runner.state.consume_base_recovery(&head));
17930        assert!(
17931            runner
17932                .state
17933                .base_sync_next_action()
17934                .contains("0/4 in episode 1, 4 lifetime")
17935        );
17936    }
17937
17938    #[cfg(unix)]
17939    #[tokio::test]
17940    async fn base_recovery_changed_head_invalidates_without_spending() {
17941        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17942        runner.state.authorize_base_sync(&task).unwrap();
17943        git::git(
17944            &runner.state.repo,
17945            &["commit", "--allow-empty", "-m", "external head change"],
17946        )
17947        .await
17948        .unwrap();
17949        runner.sync_to_base().await.unwrap();
17950        let a = runner
17951            .state
17952            .base_sync_recovery
17953            .authorizations
17954            .last()
17955            .unwrap();
17956        assert!(a.consumed);
17957        assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17958        runner.state = RunState::load(&runner.state.id).unwrap();
17959        assert!(
17960            runner
17961                .state
17962                .base_sync_next_action()
17963                .contains("Approval invalidated")
17964        );
17965        assert!(!runner.state.base_recovery_pending());
17966        assert!(runner.state.base_sync_blocked());
17967        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17968    }
17969
17970    #[cfg(unix)]
17971    #[tokio::test]
17972    async fn base_recovery_stale_fetched_base_invalidates_approval() {
17973        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17974        runner.state.driver_exited = true;
17975        runner.state.save().unwrap();
17976        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17977            .await
17978            .unwrap();
17979        runner.state = RunState::load(&runner.state.id).unwrap();
17980        advance_base_fixture(&runner).await;
17981        runner.sync_to_base().await.unwrap();
17982        let a = runner
17983            .state
17984            .base_sync_recovery
17985            .authorizations
17986            .last()
17987            .unwrap();
17988        assert!(
17989            a.consumed
17990                && a.invalidated
17991                    .as_deref()
17992                    .unwrap()
17993                    .contains("fetched base changed")
17994        );
17995        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17996        assert_eq!(
17997            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17998            head
17999        );
18000        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
18001            .await
18002            .unwrap();
18003        runner.state = RunState::load(&runner.state.id).unwrap();
18004        runner.sync_to_base().await.unwrap();
18005        assert!(!runner.state.base_sync_blocked());
18006        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18007    }
18008
18009    #[cfg(unix)]
18010    #[tokio::test]
18011    async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
18012        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18013        runner.state.authorize_base_sync(&task).unwrap();
18014        assert!(runner.state.consume_base_recovery(&head));
18015        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18016        runner.state.reserve_base_operation(&head, &tip);
18017        runner.state.save().unwrap();
18018        let scratch = runner.state.dir().join("base-sync");
18019        assert_eq!(
18020            crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18021                .await
18022                .unwrap(),
18023            crate::rebase::Rebased::Applied
18024        );
18025        runner.state = RunState::load(&runner.state.id).unwrap();
18026        runner.sync_to_base().await.unwrap();
18027        assert!(
18028            runner
18029                .state
18030                .base_sync_recovery
18031                .operations
18032                .last()
18033                .unwrap()
18034                .finished
18035        );
18036        assert_eq!(runner.state.base_episode_used(), 1);
18037        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18038        assert!(!runner.state.base_sync_blocked());
18039    }
18040
18041    #[cfg(unix)]
18042    #[tokio::test]
18043    async fn base_recovery_crash_preserves_untracked_files() {
18044        for ignored in [false, true] {
18045            let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18046            runner.state.authorize_base_sync(&task).unwrap();
18047            assert!(runner.state.consume_base_recovery(&head));
18048            let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18049            runner.state.reserve_base_operation(&head, &tip);
18050            runner.state.save().unwrap();
18051            let scratch = runner.state.dir().join("base-sync");
18052            assert_eq!(
18053                crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18054                    .await
18055                    .unwrap(),
18056                crate::rebase::Rebased::Applied
18057            );
18058            let repo = &runner.state.repo;
18059            let result = git::rev_parse(repo, "HEAD").await.unwrap();
18060            assert_ne!(result, head);
18061            if ignored {
18062                let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
18063                    .await
18064                    .unwrap();
18065                std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
18066            }
18067            std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
18068            assert!(
18069                git::git_raw(repo, &["diff", "--quiet", &head])
18070                    .await
18071                    .unwrap()
18072                    .ok()
18073            );
18074            // Resume after the branch moved, before its checkout was reset.
18075            runner.state = RunState::load(&runner.state.id).unwrap();
18076            runner.sync_to_base().await.unwrap();
18077            assert!(runner.state.base_sync_blocked());
18078            assert!(!runner.state.base_sync_recovery.operations[0].finished);
18079            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18080            assert_eq!(
18081                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18082                result
18083            );
18084            assert_eq!(
18085                std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
18086                "retained work\n"
18087            );
18088            assert!(
18089                git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
18090                    .await
18091                    .unwrap()
18092                    .ok()
18093            );
18094        }
18095    }
18096
18097    #[cfg(unix)]
18098    #[tokio::test]
18099    async fn base_recovery_crash_does_not_adopt_foreign_commits() {
18100        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18101        runner.state.authorize_base_sync(&task).unwrap();
18102        assert!(runner.state.consume_base_recovery(&head));
18103        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18104        runner.state.reserve_base_operation(&head, &tip);
18105        runner.state.save().unwrap();
18106        let scratch = runner.state.dir().join("base-sync");
18107        crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18108            .await
18109            .unwrap();
18110        git::sync_to_head(&runner.state.repo).await.unwrap();
18111        std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
18112        git::git(&runner.state.repo, &["add", "foreign"])
18113            .await
18114            .unwrap();
18115        git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
18116            .await
18117            .unwrap();
18118        let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18119        runner.state = RunState::load(&runner.state.id).unwrap();
18120        runner.sync_to_base().await.unwrap();
18121        assert!(runner.state.base_sync_blocked());
18122        assert!(
18123            !runner
18124                .state
18125                .base_sync_recovery
18126                .operations
18127                .last()
18128                .unwrap()
18129                .finished
18130        );
18131        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18132        assert_eq!(
18133            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18134            foreign_head
18135        );
18136    }
18137
18138    #[cfg(unix)]
18139    async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
18140    {
18141        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18142        fresh_base_interval_fixture(&mut runner);
18143        let repo = &runner.state.repo;
18144        git::git(repo, &["checkout", "main"]).await.unwrap();
18145        std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
18146        git::git(repo, &["add", "retained"]).await.unwrap();
18147        git::git(repo, &["commit", "-m", "upstream retained change"])
18148            .await
18149            .unwrap();
18150        git::git(repo, &["push", "origin", "main"]).await.unwrap();
18151        let tip = git::rev_parse(repo, "HEAD").await.unwrap();
18152        git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
18153        let scratch = runner.state.dir().join("base-sync");
18154        assert!(matches!(
18155            git::rebase_start(repo, &scratch, "fix/retained", &tip)
18156                .await
18157                .unwrap(),
18158            git::RebaseStart::Conflicted(_)
18159        ));
18160        runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
18161            agent: "fixture".into(),
18162            paths: vec!["retained".into()],
18163            from: Some(head.clone()),
18164            stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
18165            index_outside: None,
18166            finished: false,
18167            error: None,
18168            launcher: None,
18169            infra: None,
18170            outage: None,
18171        });
18172        (tmp, runner, head, tip, scratch)
18173    }
18174
18175    #[cfg(unix)]
18176    #[tokio::test]
18177    async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
18178        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18179        // A resource stop from before reservations existed, with resolved
18180        // contents left for the launcher's ordinary staging continuation.
18181        std::fs::write(
18182            scratch.join("retained"),
18183            "upstream and retained resolution\n",
18184        )
18185        .unwrap();
18186        runner.state.resource_block = Some(crate::run::ResourceBlock {
18187            node: "rebase".into(),
18188            kind: "repository coordination".into(),
18189            detail: "busy".into(),
18190            at: Timestamp::now(),
18191            cause: crate::run::ResourceBlockCause::Other,
18192        });
18193        runner.state.save().unwrap();
18194        // Advance the remote without moving either checkout participating
18195        // in the standing rebase.
18196        let newer = git::git(
18197            &runner.state.repo,
18198            &[
18199                "commit-tree",
18200                "main^{tree}",
18201                "-p",
18202                "main",
18203                "-m",
18204                "base advanced during wait",
18205            ],
18206        )
18207        .await
18208        .unwrap();
18209        git::git(
18210            &runner.state.repo,
18211            &[
18212                "push",
18213                "origin",
18214                &format!("{}:refs/heads/main", newer.trim()),
18215            ],
18216        )
18217        .await
18218        .unwrap();
18219        runner.state = RunState::load(&runner.state.id).unwrap();
18220        runner.state.resource_block = None;
18221        runner.sync_to_base().await.unwrap();
18222        assert!(!runner.state.base_sync_blocked());
18223        let op = &runner.state.base_sync_recovery.operations[0];
18224        assert_eq!(op.head, head);
18225        assert_eq!(op.tip, tip);
18226        assert!(op.finished);
18227        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18228        assert_eq!(runner.state.rebase_fixes.len(), 1);
18229        let latest = git::rev_parse(&runner.state.repo, "origin/main")
18230            .await
18231            .unwrap();
18232        assert_ne!(latest, tip);
18233        runner.state = RunState::load(&runner.state.id).unwrap();
18234        runner.sync_to_base().await.unwrap();
18235        assert!(!runner.state.base_sync_blocked());
18236        assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
18237        assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
18238        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
18239    }
18240
18241    #[cfg(unix)]
18242    #[tokio::test]
18243    async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
18244        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18245        runner.state.reserve_base_operation(&head, &tip);
18246        runner.state.save().unwrap();
18247        std::fs::write(
18248            scratch.join("retained"),
18249            "upstream and retained resolution\n",
18250        )
18251        .unwrap();
18252        git::git(&scratch, &["add", "retained"]).await.unwrap();
18253        git::git(
18254            &scratch,
18255            &["-c", "core.editor=true", "rebase", "--continue"],
18256        )
18257        .await
18258        .unwrap();
18259        let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18260        assert!(
18261            !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18262                .await
18263                .unwrap()
18264                .is_empty()
18265        );
18266        assert_eq!(
18267            crate::rebase::finish(
18268                &mut runner.state,
18269                &scratch,
18270                "fix/retained",
18271                &head,
18272                &tip,
18273                &["retained".into()],
18274                ""
18275            )
18276            .await
18277            .unwrap(),
18278            crate::rebase::Rebased::Applied
18279        );
18280        assert!(!scratch.exists());
18281        // Simulate losing the caller before finish_base_operation/save.
18282        runner.state = RunState::load(&runner.state.id).unwrap();
18283        let op = &runner.state.base_sync_recovery.operations[0];
18284        assert!(!op.finished);
18285        assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
18286        runner.sync_to_base().await.unwrap();
18287        assert!(!runner.state.base_sync_blocked());
18288        assert!(runner.state.base_sync_recovery.operations[0].finished);
18289        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18290        assert_eq!(runner.state.rebase_fixes.len(), 1);
18291        let winner = runner.state.winner().unwrap().clone();
18292        let wait = runner.repository_wait();
18293        assert!(
18294            verify_current_head(&mut runner.state, &winner, &wait)
18295                .await
18296                .unwrap()
18297        );
18298        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18299    }
18300
18301    #[cfg(unix)]
18302    #[tokio::test]
18303    async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
18304        for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
18305            let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18306            runner.state.reserve_base_operation(&head, &tip);
18307            runner.state.save().unwrap();
18308            std::fs::write(
18309                scratch.join("retained"),
18310                "upstream and retained resolution\n",
18311            )
18312            .unwrap();
18313            git::git(&scratch, &["add", "retained"]).await.unwrap();
18314            git::git(
18315                &scratch,
18316                &["-c", "core.editor=true", "rebase", "--continue"],
18317            )
18318            .await
18319            .unwrap();
18320            let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18321            assert!(!git::rebase_in_progress(&scratch).await);
18322            assert!(
18323                !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18324                    .await
18325                    .unwrap()
18326                    .is_empty()
18327            );
18328            match damage {
18329                Some("record") => {
18330                    let path =
18331                        git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
18332                            .await
18333                            .unwrap();
18334                    std::fs::write(scratch.join(path), "").unwrap();
18335                }
18336                Some("foreign") => {
18337                    std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
18338                    git::git(&scratch, &["add", "foreign"]).await.unwrap();
18339                    git::git(&scratch, &["commit", "-m", "foreign work"])
18340                        .await
18341                        .unwrap();
18342                }
18343                Some("dirty") => {
18344                    std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
18345                }
18346                _ => {}
18347            }
18348            let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18349            // Crash immediately after Git continuation, before finish runs.
18350            runner.state = RunState::load(&runner.state.id).unwrap();
18351            assert!(
18352                runner.state.base_sync_recovery.operations[0]
18353                    .validated_head
18354                    .is_none()
18355            );
18356            runner.sync_to_base().await.unwrap();
18357            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18358            assert_eq!(runner.state.base_episode_used(), 1);
18359            assert_eq!(runner.state.rebase_fixes.len(), 1);
18360            assert_eq!(
18361                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18362                retained_head
18363            );
18364            if damage.is_some() {
18365                assert!(runner.state.base_sync_blocked());
18366                assert!(scratch.exists());
18367                assert!(!runner.state.base_sync_recovery.operations[0].finished);
18368                assert!(
18369                    runner.state.base_sync_recovery.operations[0]
18370                        .validated_head
18371                        .is_none()
18372                );
18373            } else {
18374                assert!(!runner.state.base_sync_blocked());
18375                assert!(!scratch.exists());
18376                assert!(runner.state.base_sync_recovery.operations[0].finished);
18377                assert_eq!(
18378                    runner.state.base_sync_recovery.operations[0]
18379                        .validated_head
18380                        .as_deref(),
18381                    Some(result.as_str())
18382                );
18383                let winner = runner.state.winner().unwrap().clone();
18384                let wait = runner.repository_wait();
18385                assert!(
18386                    verify_current_head(&mut runner.state, &winner, &wait)
18387                        .await
18388                        .unwrap()
18389                );
18390                assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18391                runner.state = RunState::load(&runner.state.id).unwrap();
18392                runner.sync_to_base().await.unwrap();
18393                assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18394            }
18395        }
18396    }
18397
18398    #[cfg(unix)]
18399    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
18400        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18401        // This is an ordinary gate stop, not an authorized base-budget recovery.
18402        runner.state.prep_complete = Some(true);
18403        runner.state.base_sync = None;
18404        runner.state.base_sync_recovery = Default::default();
18405        runner.state.config.graph.gate_fix_rounds = 1;
18406        let mut review = review_round(true, 0, 1, 1, false, true);
18407        review.head = head.clone();
18408        review.verified_head = Some(head.clone());
18409        runner.state.reviews = vec![review];
18410        runner.state.gate = vec![CommandOutcome {
18411            command: "old gate".into(),
18412            code: Some(1),
18413            output_tail: "old failure".into(),
18414            duration_ms: 0,
18415            resource_blocked: false,
18416        }];
18417        runner.state.gate_head = Some(head.clone());
18418        runner.state.gate_ran = true;
18419        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
18420            agent: "fixture".into(),
18421            failed: runner.state.gate.clone(),
18422            notes: "spent".into(),
18423            committed: false,
18424            error: None,
18425        }];
18426        runner.state.land_approval = Some(crate::run::LandApproval {
18427            question: "old approval".into(),
18428            head: head.clone(),
18429        });
18430        runner.state.status = RunStatus::Blocked;
18431        (tmp, runner, head)
18432    }
18433
18434    #[cfg(unix)]
18435    #[tokio::test]
18436    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
18437        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
18438            let (tmp, mut runner, old) = resumed_gate_fixture().await;
18439            let phases = tmp.path().join("phases");
18440            runner.state.config.verify.e2e = vec![format!(
18441                "echo e2e >> '{}'; {}",
18442                phases.display(),
18443                if e2e_ok {
18444                    "true"
18445                } else {
18446                    "echo new-e2e-failure; false"
18447                }
18448            )];
18449            runner.state.config.verify.gate = vec![format!(
18450                "echo gate >> '{}'; {}",
18451                phases.display(),
18452                if gate_ok {
18453                    "true"
18454                } else {
18455                    "echo new-gate-failure; false"
18456                }
18457            )];
18458            runner.state.save().unwrap();
18459            runner.state = RunState::load(&runner.state.id).unwrap();
18460            runner.sync_to_base().await.unwrap();
18461            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18462            assert_ne!(head, old);
18463            // Reentry after Git updated but before verification was invalidated.
18464            runner.state = RunState::load(&runner.state.id).unwrap();
18465            runner.execute_graph().await.unwrap();
18466            assert_eq!(runner.state.gate_fixes.len(), 1);
18467            assert_eq!(runner.state.reviews.len(), 1);
18468            assert_eq!(runner.state.reviews[0].head, old);
18469            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18470            assert_eq!(runner.state.verification_history.len(), 1);
18471            let history = &runner.state.verification_history[0];
18472            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18473            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18474            assert!(history.archived_at.is_some());
18475            assert_eq!(history.fixes_spent, Some(1));
18476            assert_eq!(history.gate[0].output_tail, "old failure");
18477            let archive_events: Vec<_> = runner
18478                .state
18479                .events
18480                .iter()
18481                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18482                .collect();
18483            assert_eq!(archive_events.len(), 1);
18484            assert!(archive_events[0].message.contains(&short(&old)));
18485            assert!(archive_events[0].message.contains(&short(&head)));
18486            assert!(
18487                archive_events[0]
18488                    .message
18489                    .contains("1 gate-fix round(s) already spent stay spent")
18490            );
18491            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18492            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18493            if e2e_ok {
18494                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18495                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18496            } else {
18497                assert!(!runner.state.gate_ran);
18498            }
18499            assert_eq!(
18500                runner.state.status,
18501                if e2e_ok && gate_ok {
18502                    RunStatus::Ready
18503                } else {
18504                    RunStatus::Blocked
18505                }
18506            );
18507            if !e2e_ok || !gate_ok {
18508                let cause = runner.state.verification_blocking_cause().unwrap();
18509                assert!(cause.contains(&head));
18510                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18511                // A later successful progress event must not become the cause.
18512                runner.state.event("land", "rebased successfully");
18513                let notice = crate::notices::run_ended(&runner.state).unwrap();
18514                assert!(!format!("{notice:?}").contains("rebased successfully"));
18515                assert!(runner.state.merge.is_none());
18516            }
18517            let before = std::fs::read_to_string(&phases).unwrap();
18518            runner.state.save().unwrap();
18519            runner.state = RunState::load(&runner.state.id).unwrap();
18520            runner.review_loop().await.unwrap();
18521            runner.gate().await.unwrap();
18522            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18523            assert_eq!(runner.state.verification_history.len(), 1);
18524            let history = &runner.state.verification_history[0];
18525            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18526            assert!(history.archived_at.is_some());
18527            assert_eq!(history.fixes_spent, Some(1));
18528        }
18529    }
18530
18531    #[cfg(unix)]
18532    #[tokio::test]
18533    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18534        let (tmp, mut runner, head) = resumed_gate_fixture().await;
18535        let marker = tmp.path().join("gate-ran");
18536        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18537        runner.review_loop().await.unwrap();
18538        runner.gate().await.unwrap();
18539        assert_eq!(runner.state.status, RunStatus::Blocked);
18540        assert!(!marker.exists());
18541        assert!(runner.state.verification_history.is_empty());
18542        runner.state.gate_head = None;
18543        runner.gate().await.unwrap();
18544        assert!(marker.exists());
18545        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18546        assert_eq!(runner.state.verification_history.len(), 1);
18547        assert_eq!(runner.state.gate_fixes.len(), 1);
18548    }
18549
18550    #[cfg(unix)]
18551    #[tokio::test]
18552    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18553        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18554        runner.sync_to_base().await.unwrap();
18555        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18556        runner.state.reconcile_verification_head(&head);
18557        // Both crash-before-command and resource wait are unresolved evidence.
18558        for outcomes in [
18559            Vec::new(),
18560            vec![CommandOutcome {
18561                command: "true".into(),
18562                code: None,
18563                output_tail: "cache unavailable".into(),
18564                duration_ms: 0,
18565                resource_blocked: true,
18566            }],
18567        ] {
18568            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18569                head: head.clone(),
18570                passed: false,
18571                outcomes,
18572            });
18573            runner.state.gate_ran = false;
18574            runner.state.gate.clear();
18575            runner.state.save().unwrap();
18576            runner.state = RunState::load(&runner.state.id).unwrap();
18577            runner.merge().await.unwrap();
18578            assert!(runner.state.merge.is_none());
18579            runner.review_loop().await.unwrap();
18580            runner.gate().await.unwrap();
18581            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18582            assert!(runner.state.gate_allows_progress());
18583            assert_eq!(runner.state.gate_fixes.len(), 1);
18584            assert!(runner.state.merge.is_none());
18585        }
18586        // Moving the winner after a successful gate cannot publish an unchecked SHA.
18587        git::git(
18588            &runner.state.repo,
18589            &["commit", "--allow-empty", "-m", "external update"],
18590        )
18591        .await
18592        .unwrap();
18593        runner.merge().await.unwrap();
18594        assert!(runner.state.merge.is_none());
18595        assert_eq!(runner.state.status, RunStatus::Blocked);
18596    }
18597
18598    #[cfg(unix)]
18599    #[tokio::test]
18600    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18601        for waiting_on_gate in [false, true] {
18602            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18603            runner.sync_to_base().await.unwrap();
18604            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18605            runner.state.reconcile_verification_head(&head);
18606            if waiting_on_gate {
18607                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18608                    head: head.clone(),
18609                    passed: true,
18610                    outcomes: Vec::new(),
18611                });
18612            }
18613            let cache = tmp.path().join("cache");
18614            runner.state.config.verify.e2e =
18615                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18616            runner.state.config.verify.gate = vec!["true".into()];
18617            runner.state.config.graph.timeout_resource_wait = 1;
18618            let owner =
18619                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18620            let guard =
18621                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18622                    crate::cache::AcquireOutcome::Acquired(g) => g,
18623                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18624                };
18625            runner.review_loop().await.unwrap();
18626            runner.gate().await.unwrap();
18627            assert!(!runner.state.gate_ran);
18628            assert_eq!(runner.state.gate_fixes.len(), 1);
18629            assert!(runner.state.resource_block.is_some());
18630            runner.merge().await.unwrap();
18631            assert!(runner.state.merge.is_none());
18632            drop(guard);
18633            runner.state = RunState::load(&runner.state.id).unwrap();
18634            // The driver clears the previous pass's resource stop on resume.
18635            runner.state.resource_block = None;
18636            runner.review_loop().await.unwrap();
18637            runner.gate().await.unwrap();
18638            assert!(runner.state.gate_allows_progress());
18639            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18640            assert_eq!(runner.state.gate_fixes.len(), 1);
18641        }
18642    }
18643
18644    #[cfg(unix)]
18645    #[tokio::test]
18646    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18647        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18648        runner.sync_to_base().await.unwrap();
18649        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18650        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18651        // Verification passes before pre_gate and fails on the mechanically changed head.
18652        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18653        runner.review_loop().await.unwrap();
18654        runner.gate().await.unwrap();
18655        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18656        assert_ne!(before, after);
18657        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18658        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18659        assert!(!runner.state.gate_allows_progress());
18660        runner.merge().await.unwrap();
18661        assert!(runner.state.merge.is_none());
18662        assert!(
18663            runner
18664                .state
18665                .verification_history
18666                .iter()
18667                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18668        );
18669
18670        // An incomplete review is still an independent hold on a changed head.
18671        runner.state.reviews[0].clean = false;
18672        runner.state.reviews[0].answered = 0;
18673        runner.state.config.verify.e2e = vec!["true".into()];
18674        runner.review_loop().await.unwrap();
18675        runner.gate().await.unwrap();
18676        runner.merge().await.unwrap();
18677        assert_eq!(runner.state.status, RunStatus::Blocked);
18678        assert!(runner.state.merge.is_none());
18679        assert!(
18680            runner
18681                .state
18682                .verification_blocking_cause()
18683                .unwrap()
18684                .contains("review blocked")
18685        );
18686    }
18687
18688    #[cfg(unix)]
18689    #[tokio::test]
18690    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18691        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18692        runner.state.gate_head = None;
18693        runner.state.config.verify.gate =
18694            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18695        runner.review_loop().await.unwrap();
18696        runner.gate().await.unwrap();
18697        assert_ne!(
18698            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18699            old
18700        );
18701        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18702        assert!(!runner.state.gate_allows_progress());
18703        assert_eq!(runner.state.status, RunStatus::Blocked);
18704        runner.merge().await.unwrap();
18705        assert!(runner.state.merge.is_none());
18706    }
18707
18708    #[cfg(unix)]
18709    #[tokio::test]
18710    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18711        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18712        let mut old_review = review_round(true, 0, 1, 1, false, true);
18713        old_review.head = head.clone();
18714        old_review.verified_head = Some(head.clone());
18715        runner.state.reviews = vec![old_review];
18716        runner.state.gate = vec![CommandOutcome {
18717            command: "old gate".into(),
18718            code: None,
18719            output_tail: "timed out after 2400s".into(),
18720            duration_ms: 2_400_000,
18721            resource_blocked: false,
18722        }];
18723        runner.state.gate_ran = true;
18724        runner.state.gate_head = Some(head.clone());
18725        runner.state.land_approval = Some(crate::run::LandApproval {
18726            question: "old-head-approval".into(),
18727            head: head.clone(),
18728        });
18729        runner.state.authorize_base_sync(&task).unwrap();
18730        runner.sync_to_base().await.unwrap();
18731        assert!(!runner.state.base_sync_blocked());
18732        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18733        assert_ne!(head, fresh);
18734        assert_eq!(runner.state.reviews.len(), 1);
18735        assert!(!runner.state.gate_ran);
18736        let saved = &runner.state.base_sync_recovery.authorizations[0];
18737        assert_eq!(saved.reviews[0].head, head);
18738        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18739        assert!(saved.gate_ran);
18740        assert!(!saved.gate[0].ok());
18741        runner.review_loop().await.unwrap();
18742        runner.gate().await.unwrap();
18743        assert_eq!(runner.state.reviews[0].head, head);
18744        assert_eq!(
18745            runner.state.reviews[0].verified_head.as_deref(),
18746            Some(head.as_str())
18747        );
18748        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18749        assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18750        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18751        assert!(runner.state.gate_ran);
18752        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18753        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18754        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18755        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18756        assert_eq!(
18757            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18758            "retained change"
18759        );
18760    }
18761
18762    #[cfg(unix)]
18763    #[tokio::test]
18764    async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18765        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18766        let queue = crate::queue::Queue::open();
18767        let mut owner = queue.get(&task).unwrap();
18768        owner.release_operator("test-resume", None);
18769        queue.put(&mut owner).unwrap();
18770        let generation = owner.resumption.as_ref().unwrap().generation;
18771        runner.state.task_instruction = Some(owner.instruction.clone());
18772        runner.state.driver_exited = true;
18773        runner.state.save().unwrap();
18774        let home = crate::run::home();
18775        let _claim = queue.claim(&task).unwrap();
18776        owner.resumption.as_mut().unwrap().at =
18777            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18778        queue.put(&mut owner).unwrap();
18779        assert!(
18780            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18781                .await
18782                .is_err(),
18783            "an approval from before the stop grants nothing"
18784        );
18785        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18786        queue.put(&mut owner).unwrap();
18787        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18788            .await
18789            .unwrap();
18790        assert!(
18791            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18792                .await
18793                .is_err()
18794        );
18795        runner.state = RunState::load(&runner.state.id).unwrap();
18796        assert_eq!(
18797            runner.state.base_sync_recovery.authorizations[0].generation,
18798            Some(generation)
18799        );
18800        runner.sync_to_base().await.unwrap();
18801        let repo = runner.state.repo.clone();
18802        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18803        git::git(&repo, &["checkout", "main"]).await.unwrap();
18804        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18805        git::git(&repo, &["add", "."]).await.unwrap();
18806        git::git(&repo, &["commit", "-m", "move base again"])
18807            .await
18808            .unwrap();
18809        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18810        git::git(&repo, &["checkout", "fix/retained"])
18811            .await
18812            .unwrap();
18813        runner.sync_to_base().await.unwrap();
18814        assert!(!runner.state.base_sync_blocked());
18815        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18816        assert_eq!(runner.state.base_episode_used(), 2);
18817        let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18818        assert_ne!(fresh, head);
18819        assert!(!runner.state.base_recovery_pending());
18820        // Even replaying the generation with a later timestamp grants nothing.
18821        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18822        queue.put(&mut owner).unwrap();
18823        assert!(
18824            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18825                .await
18826                .is_err()
18827        );
18828        runner.state = RunState::load(&runner.state.id).unwrap();
18829        runner.sync_to_base().await.unwrap();
18830        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18831        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18832        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18833    }
18834
18835    #[cfg(unix)]
18836    #[tokio::test]
18837    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18838        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18839        runner.state.base_sync = None;
18840        runner.state.status = RunStatus::Blocked;
18841        let mut second = runner.state.candidates[0].clone();
18842        second.label = 'B';
18843        runner.state.candidates.push(second);
18844        assert!(runner.state.votes.is_empty());
18845        assert!(runner.state.deliberation.is_empty());
18846        runner.deliberate().await.unwrap();
18847        runner.vote().await.unwrap();
18848        assert_eq!(runner.state.status, RunStatus::Blocked);
18849        assert!(runner.state.votes.is_empty());
18850        assert!(runner.state.deliberation.is_empty());
18851    }
18852
18853    /// A minimal, git-free `Runner` at a given status — `settle_questions`
18854    /// reads nothing else off it.
18855    fn runner_at(status: RunStatus) -> Runner {
18856        let mut state = RunState::new(
18857            PathBuf::from("/nonexistent/repo"),
18858            "main".to_owned(),
18859            "deadbeef".to_owned(),
18860            "task".to_owned(),
18861            Config::default(),
18862        );
18863        // These fixtures test execution flow, independent of host disk capacity.
18864        state.config.disk.min_free_bytes = 0;
18865        // These fixtures share one home and save concurrently. Short clock
18866        // IDs can collide, making an ownership test read another test's run.
18867        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18868        state.id = format!(
18869            "20000101-000000-{:04x}",
18870            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18871        );
18872        state.status = status;
18873        Runner {
18874            state,
18875            roles: ResolvedRoles {
18876                implementers: Vec::new(),
18877                judges: Vec::new(),
18878                reviewers: Vec::new(),
18879                fixer: None,
18880                conductor: conductor(),
18881                implementer_roster: Vec::new(),
18882                judge_roster: Vec::new(),
18883                reviewer_roster: Vec::new(),
18884            },
18885            sem: Arc::new(Semaphore::new(1)),
18886            pause: Pause::new(),
18887            interrupt: Pause::new(),
18888        }
18889    }
18890
18891    #[tokio::test]
18892    async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18893        crate::run::pin_test_home();
18894        let temp = tempfile::tempdir().unwrap();
18895        let root = temp.path().canonicalize().unwrap();
18896        let repo = root.join("repo");
18897        std::fs::create_dir(&repo).unwrap();
18898        init_repo(&repo);
18899        git::git(&repo, &["config", "core.autocrlf", "false"])
18900            .await
18901            .unwrap();
18902        let mut runner = runner_at(RunStatus::Blocked);
18903        runner.state.repo = repo.clone();
18904        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18905        runner.state.config.agents = vec![conductor()];
18906        runner.state.config.graph.selected_implementer = Some("conductor".into());
18907        runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18908        let (shell, command, setup_output) = if cfg!(windows) {
18909            (
18910                vec!["cmd".into(), "/C".into()],
18911                "echo ready>seat-ready",
18912                "ready\r\n",
18913            )
18914        } else {
18915            (
18916                vec!["sh".into(), "-c".into()],
18917                "printf ready > seat-ready",
18918                "ready",
18919            )
18920        };
18921        runner.state.config.verify.shell = Some(shell);
18922        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18923            run: Some(command.into()),
18924            ..Default::default()
18925        }];
18926        runner.state.save().unwrap();
18927        let path = runner.state.worktree_root().join("judge-1");
18928        git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18929            .await
18930            .unwrap();
18931        std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18932        git::git(&path, &["add", "."]).await.unwrap();
18933        let tree = git::git(&path, &["write-tree"]).await.unwrap();
18934        let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18935            .await
18936            .unwrap();
18937        git::git(&path, &["checkout", "--detach", &head])
18938            .await
18939            .unwrap();
18940        let evidence = crate::fold_evidence::snapshot(
18941            &runner.state,
18942            &crate::run::home().canonicalize().unwrap(),
18943            &head,
18944            64 * 1024 * 1024,
18945        )
18946        .await
18947        .unwrap();
18948        let mut contents = BTreeMap::new();
18949        contents.insert(
18950            repo.join("logs/HEAD"),
18951            format!("{head} {head} fixture\n").into_bytes(),
18952        );
18953        let record = serde_json::json!({
18954            "version":1, "run":runner.state.id, "source":path, "head":head,
18955            "evidence":evidence, "admin":{}, "admin_contents":contents,
18956            "history":{head.clone():evidence}, "phase":"evicted"
18957        });
18958        std::fs::write(
18959            runner.state.dir().join("artifacts/eviction-judge-1.json"),
18960            serde_json::to_vec(&record).unwrap(),
18961        )
18962        .unwrap();
18963        assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18964        git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18965            .await
18966            .unwrap();
18967        git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18968        assert!(
18969            !git::git_raw(&repo, &["cat-file", "-e", &head])
18970                .await
18971                .unwrap()
18972                .ok()
18973        );
18974        let mut resumed = Runner::resume(&runner.state.id).unwrap();
18975        let error = resumed.execute().await.unwrap_err();
18976        assert!(
18977            error
18978                .to_string()
18979                .contains("selected implementation is held"),
18980            "{error:#}"
18981        );
18982        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18983        assert_eq!(
18984            std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18985            setup_output
18986        );
18987        assert_eq!(
18988            std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18989            "unrelated seat"
18990        );
18991        let error = resumed.execute().await.unwrap_err();
18992        assert!(
18993            error
18994                .to_string()
18995                .contains("selected implementation is held"),
18996            "{error:#}"
18997        );
18998        std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18999        git::git(&path, &["add", "."]).await.unwrap();
19000        let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
19001        let next = git::git(
19002            &path,
19003            &[
19004                "commit-tree",
19005                &next_tree,
19006                "-p",
19007                &head,
19008                "-m",
19009                "next review checkpoint",
19010            ],
19011        )
19012        .await
19013        .unwrap();
19014        git::git(&path, &["checkout", "--detach", &next])
19015            .await
19016            .unwrap();
19017        resumed.state.reviews.push(
19018            serde_json::from_value(serde_json::json!({
19019                "round":1, "head":next, "reviews":[]
19020            }))
19021            .unwrap(),
19022        );
19023        resumed.state.save().unwrap();
19024        let error = resumed.execute().await.unwrap_err();
19025        assert!(
19026            error
19027                .to_string()
19028                .contains("selected implementation is held"),
19029            "{error:#}"
19030        );
19031        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
19032    }
19033
19034    #[tokio::test]
19035    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
19036        crate::run::pin_test_home();
19037        let mut runner = runner_at(RunStatus::Gating);
19038        runner.state.save().unwrap();
19039        let id = runner.state.id.clone();
19040        let passes = runner.state.passes.len();
19041        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19042        let mut claiming = Box::pin(runner.claim_driver());
19043        std::future::poll_fn(|cx| {
19044            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
19045            std::task::Poll::Ready(())
19046        })
19047        .await;
19048        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
19049        drop(owner);
19050        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
19051            .await
19052            .unwrap()
19053            .unwrap();
19054        assert_eq!(execution.runner.state.passes.len(), passes + 1);
19055        assert!(execution.runner.state.resource_block.is_none());
19056        assert_eq!(
19057            execution.runner.state.liveness(false),
19058            crate::run::Liveness::Live
19059        );
19060        drop(execution);
19061        assert!(RunState::load(&id).unwrap().driver_exited);
19062    }
19063
19064    #[tokio::test]
19065    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
19066        crate::run::pin_test_home();
19067        let mut runner = runner_at(RunStatus::Gating);
19068        runner.state.save().unwrap();
19069        // run.json can commit, but the subsequent lease metadata write fails.
19070        // Cleanup must run after releasing the short ownership lock.
19071        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
19072        assert!(runner.claim_driver().await.is_err());
19073        assert!(runner.state.driver_exited);
19074        assert_eq!(runner.state.driver_pid, None);
19075        let disk = RunState::load(&runner.state.id).unwrap();
19076        assert!(disk.driver_exited);
19077        assert_eq!(disk.driver_pid, None);
19078        assert_eq!(disk.status, RunStatus::Gating);
19079    }
19080
19081    #[tokio::test]
19082    async fn an_early_graph_error_releases_its_driver() {
19083        crate::run::pin_test_home();
19084        let mut runner = runner_at(RunStatus::Blocked);
19085        runner.state.config.graph.selected_implementer = Some("selected".into());
19086        runner.state.save().unwrap();
19087        assert!(runner.execute().await.is_err());
19088        let disk = RunState::load(&runner.state.id).unwrap();
19089        assert!(disk.driver_exited);
19090        assert_eq!(disk.driver_pid, None);
19091        assert_eq!(disk.status, RunStatus::Blocked);
19092    }
19093
19094    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
19095        crate::run::MergeRepair {
19096            question: "repair-question".into(),
19097            head: "old-head".into(),
19098            pr: "https://github.com/o/r/pull/42".into(),
19099            reviews: vec![review_round(false, 1, 1, 1, false, true)],
19100            gate: vec![outcome(Some(0), false)],
19101            gate_head: Some("old-head".into()),
19102            gate_fixes: Vec::new(),
19103            pre_gate: Vec::new(),
19104            phase,
19105            failure: None,
19106        }
19107    }
19108
19109    #[tokio::test]
19110    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
19111        crate::run::pin_test_home();
19112        let mut runner = runner_at(RunStatus::Landing);
19113        runner.state.merge_repairs.push(merge_repair_request(
19114            crate::run::MergeRepairPhase::Requested,
19115        ));
19116        runner.state.save().unwrap();
19117        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
19118        assert!(
19119            runner
19120                .execute()
19121                .await
19122                .unwrap_err()
19123                .to_string()
19124                .contains("already running")
19125        );
19126        assert_eq!(
19127            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19128            crate::run::MergeRepairPhase::Requested
19129        );
19130        drop(claim);
19131        let mut disk = runner.state.clone();
19132        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
19133        disk.save().unwrap();
19134        assert!(
19135            runner
19136                .execute()
19137                .await
19138                .unwrap_err()
19139                .to_string()
19140                .contains("decision changed")
19141        );
19142        assert_eq!(
19143            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19144            crate::run::MergeRepairPhase::Complete
19145        );
19146    }
19147
19148    #[test]
19149    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
19150        crate::run::pin_test_home();
19151        let mut runner = runner_at(RunStatus::Landing);
19152        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
19153        runner.state.reviews = request.reviews.clone();
19154        runner.state.gate = request.gate.clone();
19155        runner.state.gate_ran = true;
19156        runner.state.land_approval = Some(crate::run::LandApproval {
19157            question: "old".into(),
19158            head: "old-head".into(),
19159        });
19160        runner.state.merge_repairs.push(request);
19161        runner.begin_merge_repair_review(0).unwrap();
19162        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
19163        assert!(!runner.state.gate_status().ok());
19164        assert!(runner.state.land_approval.is_none());
19165        let disk = RunState::load(&runner.state.id).unwrap();
19166        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
19167        assert!(disk.merge_repairs[0].gate[0].ok());
19168        assert_eq!(
19169            disk.merge_repairs[0].phase,
19170            crate::run::MergeRepairPhase::Reviewing
19171        );
19172        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
19173        assert!(disk.reviews.is_empty());
19174    }
19175
19176    #[tokio::test]
19177    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
19178        crate::run::pin_test_home();
19179        for phase in [
19180            crate::run::MergeRepairPhase::Fixing,
19181            crate::run::MergeRepairPhase::Requested,
19182        ] {
19183            let mut runner = runner_at(RunStatus::Landing);
19184            runner.state.config.graph.land_rounds = 0;
19185            runner.state.merge_repairs.push(merge_repair_request(phase));
19186            runner.state.save().unwrap();
19187            assert!(!runner.repair_merge_review().await.unwrap());
19188            assert_eq!(runner.state.status, RunStatus::Blocked);
19189            assert_eq!(
19190                runner.state.merge_repairs[0].phase,
19191                crate::run::MergeRepairPhase::Failed
19192            );
19193            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
19194            assert!(
19195                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
19196                    "interrupted"
19197                } else {
19198                    "budget exhausted"
19199                })
19200            );
19201            assert!(runner.state.merge.is_none());
19202        }
19203    }
19204
19205    /// `park_here` folding in the reason `Pause::park_because` recorded -
19206    /// this is what lets an operator reading a run's events tell an
19207    /// interrupt-driven park from an ordinary shutdown park.
19208    #[test]
19209    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
19210        crate::run::pin_test_home();
19211        let mut runner = runner_at(RunStatus::Implementing);
19212        let interrupt = Pause::new();
19213        runner.watch_interrupt(interrupt.clone());
19214
19215        interrupt.park_because("task a1b2 asked to run first");
19216
19217        assert!(runner.park_here().expect("park_here"));
19218        assert!(runner.state.parked);
19219        let last = runner.state.events.last().expect("a park event");
19220        assert_eq!(last.node, "park");
19221        assert!(
19222            last.message.contains("task a1b2 asked to run first"),
19223            "expected the interrupt reason in {:?}",
19224            last.message
19225        );
19226    }
19227
19228    /// The cause is stored on the pass when it parks, and the first cause a
19229    /// shared pause was given is the one that stays.
19230    #[test]
19231    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
19232        crate::run::pin_test_home();
19233        let mut runner = runner_at(RunStatus::Implementing);
19234        runner.state.begin_pass();
19235        let shutdown = Pause::new();
19236        runner.on_pause(shutdown.clone());
19237        let upgrade = ParkCause::Upgrade {
19238            from: "0.121.0".to_owned(),
19239            to: Some("0.121.1".to_owned()),
19240        };
19241        shutdown.park_for(upgrade.clone());
19242        shutdown.park_for(ParkCause::Operator);
19243
19244        assert!(runner.park_here().expect("park_here"));
19245        assert_eq!(
19246            runner.state.passes[0].end,
19247            Some(crate::run::PassEnd::Parked { cause: upgrade })
19248        );
19249        let last = runner.state.events.last().expect("a park event");
19250        assert!(last.message.contains("0.121.0"), "{}", last.message);
19251    }
19252
19253    #[tokio::test]
19254    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
19255        crate::run::pin_test_home();
19256        let mut runner = runner_at(RunStatus::Reviewing);
19257        runner.state.save().unwrap();
19258        let mut execution = runner.claim_driver().await.expect("first execution");
19259        assert_eq!(execution.runner.state.passes.len(), 1);
19260        execution
19261            .driver
19262            .finish(&mut execution.runner.state)
19263            .await
19264            .unwrap();
19265    }
19266
19267    #[tokio::test]
19268    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
19269        crate::run::pin_test_home();
19270        let mut runner = runner_at(RunStatus::Reviewing);
19271        runner.state.config.agents = vec![conductor()];
19272        runner.state.config.graph.timeout_resource_wait = 0;
19273        runner.state.save().unwrap();
19274        let id = runner.state.id.clone();
19275        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19276        assert!(runner.claim_driver().await.is_err());
19277        drop(owner);
19278        drop(runner);
19279        let mut resumed = Runner::resume(&id).unwrap();
19280        assert_eq!(resumed.state.driver_claimed, Some(false));
19281        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
19282        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
19283        let mut execution = resumed
19284            .claim_driver()
19285            .await
19286            .expect("retry initial claim after reload");
19287        assert_eq!(execution.runner.state.driver_claimed, Some(true));
19288        assert_eq!(execution.runner.state.passes.len(), 1);
19289        execution
19290            .driver
19291            .finish(&mut execution.runner.state)
19292            .await
19293            .unwrap();
19294    }
19295
19296    #[tokio::test]
19297    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
19298        crate::run::pin_test_home();
19299        let mut runner = runner_at(RunStatus::Reviewing);
19300        runner.state.config.agents = vec![conductor()];
19301        runner.state.save().unwrap();
19302        let id = runner.state.id.clone();
19303        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
19304        let mut resumed = Runner::resume(&id).unwrap();
19305        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
19306        assert!(resumed.claim_driver().await.is_err());
19307        assert!(RunState::load(&id).unwrap().passes.is_empty());
19308        drop(lease);
19309        let mut execution = resumed.claim_driver().await.unwrap();
19310        execution
19311            .driver
19312            .finish(&mut execution.runner.state)
19313            .await
19314            .unwrap();
19315    }
19316
19317    #[tokio::test]
19318    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
19319        crate::run::pin_test_home();
19320        let mut runner = runner_at(RunStatus::Reviewing);
19321        let state = &mut runner.state;
19322        state.config.agents = vec![conductor()];
19323        state.driver_claimed = None;
19324        state.driver_pid = Some(std::process::id());
19325        state.driver_started_at = None;
19326        state.begin_pass();
19327        state.save().unwrap();
19328        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
19329        std::fs::write(
19330            state.dir().join("driver.json"),
19331            serde_json::to_vec(&serde_json::json!({
19332                "pid": state.driver_pid, "started_at": null,
19333                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
19334            }))
19335            .unwrap(),
19336        )
19337        .unwrap();
19338        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
19339        drop(lease);
19340        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
19341        let mut resumed = Runner::resume(&state.id).unwrap();
19342        assert!(!resumed.state.driver_exited);
19343        let mut execution = resumed
19344            .claim_driver()
19345            .await
19346            .expect("lease proves previous execution ended");
19347        assert_eq!(execution.runner.state.passes.len(), 2);
19348        execution
19349            .driver
19350            .finish(&mut execution.runner.state)
19351            .await
19352            .unwrap();
19353    }
19354
19355    #[tokio::test]
19356    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
19357        crate::run::pin_test_home();
19358        let mut runner = runner_at(RunStatus::Reviewing);
19359        let mut json = serde_json::to_value(&runner.state).unwrap();
19360        json.as_object_mut().unwrap().remove("driver_claimed");
19361        runner.state = serde_json::from_value(json).unwrap();
19362        assert_eq!(runner.state.driver_claimed, None);
19363        runner.state.save().unwrap();
19364        let mut execution = runner
19365            .claim_driver()
19366            .await
19367            .expect("the execution lease, not a missing marker, excludes a second driver");
19368        assert_eq!(execution.runner.state.passes.len(), 1);
19369        execution
19370            .driver
19371            .finish(&mut execution.runner.state)
19372            .await
19373            .unwrap();
19374    }
19375
19376    #[tokio::test]
19377    async fn driver_admission_refuses_a_proven_live_driver() {
19378        crate::run::pin_test_home();
19379        let mut runner = runner_at(RunStatus::Reviewing);
19380        let pid = std::process::id();
19381        runner.state.driver_pid = Some(pid);
19382        runner.state.driver_started_at = crate::proc::process_started_at(pid);
19383        assert!(runner.state.driver_started_at.is_some());
19384        runner.state.save().unwrap();
19385        let error = runner
19386            .claim_driver()
19387            .await
19388            .err()
19389            .expect("a live driver must refuse");
19390        assert!(error.to_string().contains("live driver"), "{error:#}");
19391        let saved = RunState::load(&runner.state.id).unwrap();
19392        assert_eq!(saved.driver_pid, Some(pid));
19393        assert_eq!(saved.status, RunStatus::Reviewing);
19394        assert!(saved.passes.is_empty());
19395    }
19396
19397    #[test]
19398    fn parking_resource_halt_persists_the_pass_once() {
19399        crate::run::pin_test_home();
19400        let mut runner = runner_at(RunStatus::Reviewing);
19401        runner.state.begin_pass();
19402        runner.state.resource_block = Some(crate::run::ResourceBlock {
19403            node: "fix".into(),
19404            kind: "shared build cache".into(),
19405            detail: "cancelled".into(),
19406            at: Timestamp::now(),
19407            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
19408        });
19409        let cause = ParkCause::Upgrade {
19410            from: "old".into(),
19411            to: Some("new".into()),
19412        };
19413        runner.pause.park_for(cause.clone());
19414        // Both direct fixer exits and graph resource boundaries must complete
19415        // parking, rather than returning a stale Reviewing record.
19416        runner.stop_for_resource().unwrap();
19417        assert!(runner.resource_halt().unwrap());
19418        assert!(runner.park_here().unwrap());
19419        let saved = RunState::load(&runner.state.id).unwrap();
19420        assert!(saved.parked);
19421        assert_eq!(saved.status, RunStatus::Reviewing);
19422        assert!(saved.passes[0].ended_at.is_some());
19423        assert_eq!(
19424            saved.passes[0].end,
19425            Some(crate::run::PassEnd::Parked {
19426                cause: cause.clone()
19427            })
19428        );
19429        assert_eq!(
19430            saved.resource_block.unwrap().cause,
19431            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19432        );
19433        assert_eq!(
19434            saved
19435                .events
19436                .iter()
19437                .filter(|event| event.node == "park")
19438                .count(),
19439            1
19440        );
19441    }
19442
19443    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
19444    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
19445    /// cleared) must not make a *different* run - one only watching its own,
19446    /// unshared interrupt `Pause` - see itself as parked. If a future change
19447    /// ever collapsed these back into one handle, the interrupt scheduler
19448    /// would park every run for the rest of the daemon's life, not just the
19449    /// one it meant to interrupt.
19450    #[test]
19451    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19452        crate::run::pin_test_home();
19453        let mut runner = runner_at(RunStatus::Implementing);
19454        let shutdown = Pause::new();
19455        runner.on_pause(shutdown.clone());
19456        let interrupt = Pause::new();
19457        runner.watch_interrupt(interrupt.clone());
19458
19459        // Nobody has asked for anything yet.
19460        assert!(!runner.park_here().expect("park_here"));
19461        assert!(!runner.state.parked);
19462
19463        // Only the interrupt handle fires; the shutdown handle stays clear.
19464        interrupt.park_because("test");
19465        assert!(!shutdown.parked());
19466        assert!(runner.park_here().expect("park_here"));
19467    }
19468
19469    /// The property every prior attempt at this feature failed to pin down:
19470    /// asking a run to park while one of its nodes has a real, in-flight
19471    /// async operation running (an agent call, in production) must not cut
19472    /// that operation short. `park_here` is only ever consulted *between*
19473    /// `execute`'s node calls - see its own doc - so nothing inside a node
19474    /// can observe a park request until the node itself returns. This proves
19475    /// that structurally, with real `tokio` concurrency and a channel
19476    /// handshake (never a sleep, which would only prove "usually", not
19477    /// "cannot"): the "node" below reports that it has genuinely started,
19478    /// and only then is the park requested; the node still has to be told to
19479    /// finish before `park_here` is ever called, exactly mirroring every
19480    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
19481    /// in `execute`.
19482    #[tokio::test]
19483    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19484        crate::run::pin_test_home();
19485        let mut runner = runner_at(RunStatus::Implementing);
19486        let interrupt = Pause::new();
19487        runner.watch_interrupt(interrupt.clone());
19488
19489        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19490        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19491
19492        // Stands in for one node's in-flight agent call: it proves it has
19493        // genuinely started, then blocks - exactly as a spawned CLI process
19494        // does - until told to finish.
19495        let node = async move {
19496            started_tx.send(()).expect("send started");
19497            finish_rx.await.expect("recv finish");
19498            "node finished"
19499        };
19500
19501        let interrupter = async move {
19502            started_rx.await.expect("recv started");
19503            // The call is now genuinely in flight. Ask it to park.
19504            interrupt.park_because("higher-priority task waiting");
19505            // Nothing the node does can observe this yet - there is no
19506            // check inside it, by construction - so let the executor run
19507            // anything pending and then let the node finish on its own.
19508            tokio::task::yield_now().await;
19509            finish_tx.send(()).expect("send finish");
19510        };
19511
19512        let (node_result, ()) = tokio::join!(node, interrupter);
19513        assert_eq!(
19514            node_result, "node finished",
19515            "the in-flight call ran to completion"
19516        );
19517
19518        // Only now, at the boundary the real `execute` would check right
19519        // after this node, does the park take effect.
19520        assert!(runner.park_here().expect("park_here"));
19521        assert!(runner.state.parked);
19522    }
19523
19524    /// A run parked mid-competition carries every field it had accumulated
19525    /// through the exact same disk round-trip an ordinary resume uses -
19526    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
19527    /// Nothing about parking for an interrupt is a special case of that path;
19528    /// this is what proves it rather than assuming it.
19529    #[test]
19530    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19531        crate::run::pin_test_home();
19532        let mut runner = runner_at(RunStatus::Judging);
19533        // `Runner::resume` re-resolves roles from the saved config, which
19534        // refuses an empty roster - give it the same minimal one `conductor`
19535        // itself uses.
19536        runner.state.config.agents = vec![conductor()];
19537        runner.state.candidates = vec![Candidate {
19538            index: 0,
19539            label: 'A',
19540            agent: "alpha".to_owned(),
19541            branch: "magi/x/A".to_owned(),
19542            worktree: PathBuf::from("/nonexistent/worktree"),
19543            summary: "did the thing".to_owned(),
19544            stat: "1 file changed".to_owned(),
19545            files: 1,
19546            commits: 1,
19547            empty: false,
19548            failed: None,
19549            verified_noop: None,
19550            duration_ms: 1234,
19551            folded: false,
19552        }];
19553        let run_id = runner.state.id.clone();
19554
19555        let interrupt = Pause::new();
19556        runner.watch_interrupt(interrupt.clone());
19557        interrupt.park_because("task c3d4 asked to run first");
19558        assert!(runner.park_here().expect("park_here"));
19559
19560        let resumed = Runner::resume(&run_id).expect("resume");
19561        assert_eq!(resumed.state.candidates.len(), 1);
19562        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19563        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19564        assert_eq!(resumed.state.status, runner.state.status);
19565        assert!(
19566            resumed.state.parked,
19567            "still parked until `execute` actually walks the graph again"
19568        );
19569        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19570    }
19571
19572    /// A fresh open question on `run`, stored and handed back for assertions.
19573    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19574        let mut q = ask::Question::new(
19575            run.to_owned(),
19576            "implement".to_owned(),
19577            "impl-A".to_owned(),
19578            "Which storage backend should the cache use?".to_owned(),
19579            String::new(),
19580            vec!["SQLite".to_owned(), "Redis".to_owned()],
19581        );
19582        store.put(&mut q).unwrap();
19583        q
19584    }
19585
19586    #[test]
19587    fn a_failed_runs_open_question_is_abandoned() {
19588        ask_test_home();
19589        let store = ask::Questions::open();
19590        let mut runner = runner_at(RunStatus::Failed);
19591        let run = runner.state.id.clone();
19592        let q = ask_open_question(&store, &run);
19593
19594        runner.settle_questions();
19595
19596        let back = store.get(&q.id).unwrap();
19597        assert!(
19598            !back.status.open(),
19599            "the seat that asked died with the run; nobody is left to read an answer"
19600        );
19601        assert!(
19602            back.detail.contains(&run) && back.detail.contains("failed"),
19603            "the reason names what the run became, not just that it is gone: {}",
19604            back.detail
19605        );
19606    }
19607
19608    #[test]
19609    fn a_merged_runs_open_question_is_abandoned_too() {
19610        ask_test_home();
19611        let store = ask::Questions::open();
19612        // A run that finishes cleanly still leaves nobody to read an answer -
19613        // this is not only a failure-path cleanup.
19614        for status in [RunStatus::Merged, RunStatus::Ready] {
19615            let mut runner = runner_at(status);
19616            let run = runner.state.id.clone();
19617            let q = ask_open_question(&store, &run);
19618
19619            runner.settle_questions();
19620
19621            let back = store.get(&q.id).unwrap();
19622            assert!(
19623                !back.status.open(),
19624                "{status:?} run's question must not outlive the run"
19625            );
19626        }
19627    }
19628
19629    #[test]
19630    fn a_still_resumable_runs_open_question_is_left_alone() {
19631        ask_test_home();
19632        let store = ask::Questions::open();
19633        // `Blocked` and `Stalled` can still be resumed — the candidates, the
19634        // review round and the seat sessions are all still on disk — so a
19635        // question asked mid-round may yet get a real answer from a real
19636        // resume. Sweeping it here would be exactly the failure mode this
19637        // whole feature exists to avoid on the other side.
19638        for status in [RunStatus::Blocked, RunStatus::Stalled] {
19639            let mut runner = runner_at(status);
19640            let run = runner.state.id.clone();
19641            let q = ask_open_question(&store, &run);
19642
19643            runner.settle_questions();
19644
19645            let back = store.get(&q.id).unwrap();
19646            assert!(
19647                back.status.open(),
19648                "{status:?} is still alive; the question must still be waiting"
19649            );
19650        }
19651    }
19652
19653    #[test]
19654    fn settle_questions_never_touches_an_already_answered_question() {
19655        ask_test_home();
19656        let store = ask::Questions::open();
19657        let mut runner = runner_at(RunStatus::Failed);
19658        let run = runner.state.id.clone();
19659        let mut q = ask_open_question(&store, &run);
19660        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19661            .unwrap();
19662        store.put(&mut q).unwrap();
19663
19664        // Called twice, the way a crash-recovered daemon reclaim and the
19665        // graph's own cleanup both can for the same run — `abandon_for_run`
19666        // only ever touches what is still open, so this must be inert both
19667        // times, not merely the second.
19668        runner.settle_questions();
19669        runner.settle_questions();
19670
19671        let back = store.get(&q.id).unwrap();
19672        assert_eq!(
19673            back.status,
19674            ask::QuestionStatus::Answered,
19675            "a real answer is a decision on record, never overwritten by a sweep"
19676        );
19677    }
19678
19679    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
19680    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
19681    /// without merging, whose winner is still the operator's answer to read.
19682    /// Nothing previously called `fold_run` itself with a real `tally`, so
19683    /// this is the first test to pin down the one distinction the whole
19684    /// automatic-fold feature depends on: the winner's worktree and branch
19685    /// must survive, everything else sharing the run's worktree bay - a
19686    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
19687    /// second sweep treats every non-winner directory under the bay alike -
19688    /// must not.
19689    #[tokio::test]
19690    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19691        crate::run::pin_test_home();
19692        let tmp = tempfile::tempdir().expect("tempdir");
19693        let repo = tmp.path().join("repo");
19694        std::fs::create_dir_all(&repo).unwrap();
19695        init_repo(&repo);
19696
19697        let mut config = Config::default();
19698        config.graph.worktree_root = Some(tmp.path().join("wt"));
19699
19700        let mut state = RunState::new(
19701            repo.clone(),
19702            "main".to_owned(),
19703            "deadbeef".to_owned(),
19704            "task".to_owned(),
19705            config,
19706        );
19707        let root = state.worktree_root();
19708        let wt_a = root.join("cand-A");
19709        let wt_b = root.join("cand-B");
19710        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19711            .await
19712            .expect("worktree A");
19713        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19714            .await
19715            .expect("worktree B");
19716
19717        state.candidates = vec![
19718            Candidate {
19719                index: 0,
19720                label: 'A',
19721                agent: "alpha".to_owned(),
19722                branch: "magi/x/A".to_owned(),
19723                worktree: wt_a.clone(),
19724                summary: String::new(),
19725                stat: String::new(),
19726                files: 0,
19727                commits: 0,
19728                empty: false,
19729                failed: None,
19730                verified_noop: None,
19731                duration_ms: 0,
19732                folded: false,
19733            },
19734            Candidate {
19735                index: 1,
19736                label: 'B',
19737                agent: "beta".to_owned(),
19738                branch: "magi/x/B".to_owned(),
19739                worktree: wt_b.clone(),
19740                summary: String::new(),
19741                stat: String::new(),
19742                files: 0,
19743                commits: 0,
19744                empty: false,
19745                failed: None,
19746                verified_noop: None,
19747                duration_ms: 0,
19748                folded: false,
19749            },
19750        ];
19751        state.tally = Some(Tally {
19752            first_choice: BTreeMap::from([('A', 1)]),
19753            borda: BTreeMap::new(),
19754            winner: 'A',
19755            rankings: 1,
19756            unanimous_initial: true,
19757            deliberated: false,
19758            changed_votes: 0,
19759            unanimous_final: true,
19760            tie_break: None,
19761            judges: 1,
19762            present: 1,
19763            quorum: 1,
19764            met_quorum: true,
19765            uncontested: None,
19766        });
19767        state.status = RunStatus::Ready;
19768
19769        fold_run(&mut state, false, &crate::run::home())
19770            .await
19771            .expect("fold_run");
19772
19773        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19774        assert!(
19775            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19776            "the unmerged winner's branch survives"
19777        );
19778        assert!(
19779            !state.candidates[0].folded,
19780            "the winner is not marked folded"
19781        );
19782
19783        assert!(!wt_b.exists(), "the loser's worktree is removed");
19784        assert!(
19785            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19786            "the loser's branch is removed"
19787        );
19788        assert!(state.candidates[1].folded, "the loser is marked folded");
19789    }
19790
19791    #[tokio::test]
19792    #[cfg(unix)]
19793    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19794        use std::os::fd::AsRawFd as _;
19795        use std::time::Duration;
19796
19797        let tmp = tempfile::tempdir().unwrap();
19798        let repo = tmp.path().join("repo");
19799        std::fs::create_dir_all(&repo).unwrap();
19800        init_repo(&repo);
19801        let common = repo.join(".git");
19802        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19803        let home = tmp.path().join("home");
19804        let mut config = Config::default();
19805        config.coordination = crate::config::test_coordination();
19806        config.graph.worktree_root = Some(tmp.path().join("wt"));
19807        config.graph.timeout_resource_wait = 1;
19808        let mut state = RunState::new(
19809            repo.clone(),
19810            "main".into(),
19811            "deadbeef".into(),
19812            "task".into(),
19813            config,
19814        );
19815        let root = state.worktree_root();
19816        let candidate = root.join("cand-A");
19817        let reviewer = root.join("review-1");
19818        let branch = state.branch_for('A');
19819        git::worktree_add_branch(&repo, &candidate, &branch, "main")
19820            .await
19821            .unwrap();
19822        git::worktree_add_detached(&repo, &reviewer, "main")
19823            .await
19824            .unwrap();
19825        git::acquire_worktree_config(&repo).await.unwrap();
19826        state.enabled_worktree_config = true;
19827        state.status = RunStatus::Ready;
19828        state.candidates.push(Candidate {
19829            index: 0,
19830            label: 'A',
19831            agent: "implement".into(),
19832            branch: branch.clone(),
19833            worktree: candidate.clone(),
19834            summary: String::new(),
19835            stat: String::new(),
19836            files: 0,
19837            commits: 0,
19838            empty: false,
19839            failed: None,
19840            verified_noop: None,
19841            duration_ms: 0,
19842            folded: false,
19843        });
19844        state.save_under(&home).unwrap();
19845        let config_before = std::fs::read(common.join("config")).unwrap();
19846        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19847            .await
19848            .unwrap();
19849        // A separate OS handle bypasses the launcher's local gate, exactly as
19850        // an external updater does, and must still prevent all folding work.
19851        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19852        assert_eq!(
19853            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19854            0
19855        );
19856        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19857        assert!(
19858            tokio::time::timeout(Duration::from_millis(200), &mut fold)
19859                .await
19860                .is_err()
19861        );
19862        assert!(candidate.exists());
19863        assert!(reviewer.exists());
19864        assert!(git::branch_exists(&repo, &branch).await.unwrap());
19865        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19866        assert_eq!(
19867            git::git(&repo, &["worktree", "list", "--porcelain"])
19868                .await
19869                .unwrap(),
19870            worktrees_before
19871        );
19872        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19873            .await
19874            .unwrap()
19875            .unwrap_err();
19876        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19877        drop(fold);
19878        assert!(candidate.exists() && reviewer.exists());
19879        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19880        assert_eq!(
19881            state.resource_block.as_ref().unwrap().kind,
19882            "repository coordination"
19883        );
19884        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19885        drop(external);
19886        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19887        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19888            .await
19889            .unwrap()
19890            .unwrap();
19891        drop(fold);
19892        assert!(removed.contains(&branch));
19893        assert!(!root.exists());
19894        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19895        assert!(state.candidates[0].folded);
19896        assert!(!state.enabled_worktree_config);
19897        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19898        let enabled = git::git_raw(
19899            &repo,
19900            &["config", "--local", "--get", "extensions.worktreeConfig"],
19901        )
19902        .await
19903        .unwrap();
19904        assert!(!enabled.ok(), "the last config reference was released");
19905    }
19906
19907    /// A branch handed to a later run is that run's (and its pull request's):
19908    /// folding the run that released it must not delete it.
19909    #[tokio::test]
19910    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19911        let tmp = tempfile::tempdir().expect("tempdir");
19912        let repo = tmp.path().join("repo");
19913        std::fs::create_dir_all(&repo).unwrap();
19914        init_repo(&repo);
19915        let home = tmp.path().join("home");
19916
19917        let mut config = Config::default();
19918        config.graph.worktree_root = Some(tmp.path().join("wt"));
19919        let mut state = RunState::new(
19920            repo.clone(),
19921            "main".to_owned(),
19922            "deadbeef".to_owned(),
19923            "task".to_owned(),
19924            config,
19925        );
19926        // The worktree is already gone (released); the branch survives.
19927        git::git(&repo, &["branch", "magi/x/A", "main"])
19928            .await
19929            .expect("branch");
19930        state.candidates = vec![Candidate {
19931            index: 0,
19932            label: 'A',
19933            agent: "alpha".to_owned(),
19934            branch: "magi/x/A".to_owned(),
19935            worktree: state.worktree_root().join("cand-A"),
19936            summary: String::new(),
19937            stat: String::new(),
19938            files: 0,
19939            commits: 0,
19940            empty: false,
19941            failed: None,
19942            verified_noop: None,
19943            duration_ms: 0,
19944            folded: true,
19945        }];
19946        state.released_to = Some("20260901-000000-new1".to_owned());
19947        state.released_branches = vec!["magi/x/A".to_owned()];
19948
19949        fold_run(&mut state, true, &home).await.expect("fold_run");
19950        state.status = RunStatus::Merged;
19951        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19952        state.merge = Some(crate::run::MergeOutcome {
19953            mode: MergeMode::Pr,
19954            ok: true,
19955            detail: "confirmed merged".into(),
19956            empty: false,
19957        });
19958        fold_run(&mut state, true, &home)
19959            .await
19960            .expect("merged fold");
19961
19962        assert!(
19963            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19964            "the handed-over branch survives a fold"
19965        );
19966    }
19967
19968    /// A winner with nothing ahead of the base is caught before `gh` is ever
19969    /// asked for a pull request, and the message carries what the task's
19970    /// references resolved to.
19971    #[tokio::test]
19972    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19973        let tmp = tempfile::tempdir().expect("tempdir");
19974        let repo = tmp.path().join("repo");
19975        std::fs::create_dir_all(&repo).unwrap();
19976        init_repo(&repo);
19977        let run = |args: &[&str]| {
19978            let out = std::process::Command::new("git")
19979                .quiet()
19980                .args(args)
19981                .current_dir(&repo)
19982                .output()
19983                .expect("spawn git");
19984            assert!(out.status.success(), "git {args:?}");
19985        };
19986        run(&["branch", "magi/x/A"]);
19987        run(&["checkout", "-q", "-b", "magi/x/B"]);
19988        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19989        run(&["add", "-A"]);
19990        run(&["commit", "-q", "-m", "work"]);
19991        run(&["checkout", "-q", "main"]);
19992
19993        // A pull request is compared against the remote's base, so the
19994        // fixture needs one. Before it exists nothing can be read, and the
19995        // answer must be "not empty".
19996        let probe = RunState::new(
19997            repo.clone(),
19998            "main".to_owned(),
19999            "deadbeef".to_owned(),
20000            "task".to_owned(),
20001            Config::default(),
20002        );
20003        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
20004        let bare = tmp.path().join("origin.git");
20005        let out = std::process::Command::new("git")
20006            .quiet()
20007            .args(["init", "-q", "--bare"])
20008            .arg(&bare)
20009            .output()
20010            .expect("spawn git");
20011        assert!(out.status.success(), "git init --bare");
20012        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
20013        run(&["push", "-q", "origin", "main"]);
20014
20015        let mut state = RunState::new(
20016            repo.clone(),
20017            "main".to_owned(),
20018            "deadbeef".to_owned(),
20019            "task".to_owned(),
20020            Config::default(),
20021        );
20022        state.seeds = vec![refs::Seed {
20023            token: "magi/27b2/A".to_owned(),
20024            kind: refs::SeedKind::Unresolved,
20025            sha: String::new(),
20026            branch: true,
20027            detail: "no branch or commit named magi/27b2/A".to_owned(),
20028        }];
20029
20030        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
20031        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
20032        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
20033        let detail = empty_candidate_detail(&state, "main");
20034        assert!(detail.starts_with("empty candidate"), "{detail}");
20035        assert!(!detail.contains("magi/27b2/A"), "{detail}");
20036        assert!(detail.contains("--start-from"), "{detail}");
20037    }
20038
20039    /// `status == Ready` used to be read as "this is the harmless
20040    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
20041    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
20042    /// run's PR was closed without merging — and reentering `merge` with
20043    /// `mode` still `Pr` does not know the difference, so it pushed and
20044    /// opened a second pull request. `mode == Local` reproduces the same
20045    /// blind spot without a network call: reentry must not attempt another
20046    /// git merge once this node has already recorded an outcome.
20047    #[tokio::test]
20048    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
20049        let tmp = tempfile::tempdir().expect("tempdir");
20050        let repo = tmp.path().join("repo");
20051        std::fs::create_dir_all(&repo).unwrap();
20052        init_repo(&repo);
20053
20054        let mut config = Config::default();
20055        config.merge.mode = MergeMode::Local;
20056
20057        let mut state = RunState::new(
20058            repo.clone(),
20059            "main".to_owned(),
20060            "deadbeef".to_owned(),
20061            "task".to_owned(),
20062            config,
20063        );
20064        state.candidates = vec![Candidate {
20065            index: 0,
20066            label: 'A',
20067            agent: "alpha".to_owned(),
20068            branch: "does-not-exist".to_owned(),
20069            worktree: repo.clone(),
20070            summary: String::new(),
20071            stat: String::new(),
20072            files: 0,
20073            commits: 0,
20074            empty: false,
20075            failed: None,
20076            verified_noop: None,
20077            duration_ms: 0,
20078            folded: false,
20079        }];
20080        state.tally = Some(Tally {
20081            first_choice: BTreeMap::from([('A', 1)]),
20082            borda: BTreeMap::new(),
20083            winner: 'A',
20084            rankings: 1,
20085            unanimous_initial: true,
20086            deliberated: false,
20087            changed_votes: 0,
20088            unanimous_final: true,
20089            tie_break: None,
20090            judges: 0,
20091            present: 0,
20092            quorum: 0,
20093            met_quorum: true,
20094            uncontested: Some("only candidate A produced a change".to_owned()),
20095        });
20096        state.reviews = vec![ReviewRound {
20097            round: 1,
20098            head: "deadbeef".to_owned(),
20099            verified_head: None,
20100            verified_at: None,
20101            reviews: Vec::new(),
20102            e2e: Vec::new(),
20103            fix: None,
20104            blocking: 0,
20105            answered: 0,
20106            expected: 0,
20107            clean: true,
20108            verify_retried: false,
20109            e2e_deferred: false,
20110            e2e_defer_reason: None,
20111            progressed: false,
20112            vote_split: false,
20113            reconsideration: Vec::new(),
20114            verdict: None,
20115        }];
20116        state.gate = vec![CommandOutcome {
20117            command: "test".to_owned(),
20118            code: Some(0),
20119            output_tail: String::new(),
20120            duration_ms: 0,
20121            resource_blocked: false,
20122        }];
20123        state.gate_ran = true;
20124        // Reached its conclusion already — e.g. `land` closing the PR without
20125        // merging it, which (like the honest `MergeMode::None` path) leaves
20126        // `status` at `Ready`. The recorded outcome is what actually marks
20127        // this node done.
20128        state.status = RunStatus::Ready;
20129        state.merge = Some(MergeOutcome {
20130            mode: MergeMode::Local,
20131            ok: false,
20132            detail: "already concluded".to_owned(),
20133            empty: false,
20134        });
20135
20136        let mut runner = Runner {
20137            state,
20138            roles: ResolvedRoles {
20139                implementers: Vec::new(),
20140                judges: Vec::new(),
20141                reviewers: Vec::new(),
20142                fixer: None,
20143                conductor: conductor(),
20144                implementer_roster: Vec::new(),
20145                judge_roster: Vec::new(),
20146                reviewer_roster: Vec::new(),
20147            },
20148            sem: Arc::new(Semaphore::new(1)),
20149            pause: Pause::new(),
20150            interrupt: Pause::new(),
20151        };
20152
20153        runner.merge().await.expect("merge");
20154
20155        assert_eq!(
20156            runner.state.status,
20157            RunStatus::Ready,
20158            "a concluded run's status must not change on reentry"
20159        );
20160        assert_eq!(
20161            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
20162            Some("already concluded"),
20163            "merge must not run again once the node already recorded an outcome"
20164        );
20165    }
20166
20167    /// `gate` leaves `state.gate_ran` false both before it has ever run and
20168    /// when its last attempt was resource-blocked (the shared build cache
20169    /// could not be acquired or confirmed fresh in time - see
20170    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
20171    /// `Vec` this also leaves behind used to read as "nothing failed" and let
20172    /// a run merge a tree the gate never actually checked - exactly the case
20173    /// a contended cache produces on every retry until it clears. `merge`
20174    /// must refuse until `gate` has actually recorded an attempt.
20175    #[tokio::test]
20176    async fn merge_refuses_a_gate_that_has_not_actually_run() {
20177        let tmp = tempfile::tempdir().expect("tempdir");
20178        let repo = tmp.path().join("repo");
20179        std::fs::create_dir_all(&repo).unwrap();
20180        init_repo(&repo);
20181
20182        let mut config = Config::default();
20183        config.merge.mode = MergeMode::Local;
20184
20185        let mut state = RunState::new(
20186            repo.clone(),
20187            "main".to_owned(),
20188            "deadbeef".to_owned(),
20189            "task".to_owned(),
20190            config,
20191        );
20192        state.candidates = vec![Candidate {
20193            index: 0,
20194            label: 'A',
20195            agent: "alpha".to_owned(),
20196            branch: "does-not-exist".to_owned(),
20197            worktree: repo.clone(),
20198            summary: String::new(),
20199            stat: String::new(),
20200            files: 0,
20201            commits: 0,
20202            empty: false,
20203            failed: None,
20204            verified_noop: None,
20205            duration_ms: 0,
20206            folded: false,
20207        }];
20208        state.tally = Some(Tally {
20209            first_choice: BTreeMap::from([('A', 1)]),
20210            borda: BTreeMap::new(),
20211            winner: 'A',
20212            rankings: 1,
20213            unanimous_initial: true,
20214            deliberated: false,
20215            changed_votes: 0,
20216            unanimous_final: true,
20217            tie_break: None,
20218            judges: 0,
20219            present: 0,
20220            quorum: 0,
20221            met_quorum: true,
20222            uncontested: Some("only candidate A produced a change".to_owned()),
20223        });
20224        state.reviews = vec![ReviewRound {
20225            round: 1,
20226            head: "deadbeef".to_owned(),
20227            verified_head: None,
20228            verified_at: None,
20229            reviews: Vec::new(),
20230            e2e: Vec::new(),
20231            fix: None,
20232            blocking: 0,
20233            answered: 0,
20234            expected: 0,
20235            clean: true,
20236            verify_retried: false,
20237            e2e_deferred: false,
20238            e2e_defer_reason: None,
20239            progressed: false,
20240            vote_split: false,
20241            reconsideration: Vec::new(),
20242            verdict: None,
20243        }];
20244        // The point: `gate` has not recorded anything yet.
20245        state.gate = Vec::new();
20246        state.gate_ran = false;
20247        state.status = RunStatus::Gating;
20248
20249        let mut runner = Runner {
20250            state,
20251            roles: ResolvedRoles {
20252                implementers: Vec::new(),
20253                judges: Vec::new(),
20254                reviewers: Vec::new(),
20255                fixer: None,
20256                conductor: conductor(),
20257                implementer_roster: Vec::new(),
20258                judge_roster: Vec::new(),
20259                reviewer_roster: Vec::new(),
20260            },
20261            sem: Arc::new(Semaphore::new(1)),
20262            pause: Pause::new(),
20263            interrupt: Pause::new(),
20264        };
20265
20266        runner.merge().await.expect("merge");
20267
20268        assert!(
20269            runner.state.merge.is_none(),
20270            "an empty gate must never be read as a passing one: {:?}",
20271            runner.state.merge
20272        );
20273    }
20274
20275    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
20276    /// commands configured and `merge.mode` is `none` (a review-only run).
20277    /// `gate` must still record a real attempt — zero commands, vacuously
20278    /// passed — rather than leaving `state.gate` empty in a way `merge`
20279    /// cannot tell apart from "never ran"; otherwise the run reaches
20280    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
20281    #[tokio::test]
20282    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
20283        let tmp = tempfile::tempdir().expect("tempdir");
20284        let repo = tmp.path().join("repo");
20285        std::fs::create_dir_all(&repo).unwrap();
20286        init_repo(&repo);
20287
20288        // Default config: `verify.gate` empty, `merge.mode` is `none`.
20289        let config = Config::default();
20290
20291        let mut state = RunState::new(
20292            repo.clone(),
20293            "main".to_owned(),
20294            "deadbeef".to_owned(),
20295            "task".to_owned(),
20296            config,
20297        );
20298        state.candidates = vec![Candidate {
20299            index: 0,
20300            label: 'A',
20301            agent: "alpha".to_owned(),
20302            branch: "does-not-exist".to_owned(),
20303            worktree: repo.clone(),
20304            summary: String::new(),
20305            stat: String::new(),
20306            files: 0,
20307            commits: 0,
20308            empty: false,
20309            failed: None,
20310            verified_noop: None,
20311            duration_ms: 0,
20312            folded: false,
20313        }];
20314        state.tally = Some(Tally {
20315            first_choice: BTreeMap::from([('A', 1)]),
20316            borda: BTreeMap::new(),
20317            winner: 'A',
20318            rankings: 1,
20319            unanimous_initial: true,
20320            deliberated: false,
20321            changed_votes: 0,
20322            unanimous_final: true,
20323            tie_break: None,
20324            judges: 0,
20325            present: 0,
20326            quorum: 0,
20327            met_quorum: true,
20328            uncontested: Some("only candidate A produced a change".to_owned()),
20329        });
20330        state.reviews = vec![ReviewRound {
20331            round: 1,
20332            head: "deadbeef".to_owned(),
20333            verified_head: None,
20334            verified_at: None,
20335            reviews: Vec::new(),
20336            e2e: Vec::new(),
20337            fix: None,
20338            blocking: 0,
20339            answered: 0,
20340            expected: 0,
20341            clean: true,
20342            verify_retried: false,
20343            e2e_deferred: false,
20344            e2e_defer_reason: None,
20345            progressed: false,
20346            vote_split: false,
20347            reconsideration: Vec::new(),
20348            verdict: None,
20349        }];
20350
20351        let mut runner = Runner {
20352            state,
20353            roles: ResolvedRoles {
20354                implementers: Vec::new(),
20355                judges: Vec::new(),
20356                reviewers: Vec::new(),
20357                fixer: None,
20358                conductor: conductor(),
20359                implementer_roster: Vec::new(),
20360                judge_roster: Vec::new(),
20361                reviewer_roster: Vec::new(),
20362            },
20363            sem: Arc::new(Semaphore::new(1)),
20364            pause: Pause::new(),
20365            interrupt: Pause::new(),
20366        };
20367
20368        runner.gate().await.expect("gate");
20369        assert!(
20370            runner.state.gate_ran,
20371            "zero configured commands is still a real attempt, not an unrun gate"
20372        );
20373        assert!(runner.state.gate.is_empty());
20374        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
20375        assert_ne!(
20376            runner.state.status,
20377            RunStatus::Blocked,
20378            "a gate with nothing to check must not read as failed"
20379        );
20380
20381        runner.merge().await.expect("merge");
20382        assert_eq!(
20383            runner.state.status,
20384            RunStatus::Ready,
20385            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
20386        );
20387    }
20388
20389    /// `Config::cache_dir` is derived from `verify.e2e` as well as
20390    /// `verify.gate` (so the e2e leg and the final gate never build against
20391    /// different directories). With zero `verify.gate` commands but a
20392    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
20393    /// that lease before discovering it had nothing to run - so a repo with
20394    /// no gate commands could come back `resource_blocked` (and therefore
20395    /// still `gate_ran == false`) on nothing but an unrelated run holding the
20396    /// cache, exactly the contention this run's own zero commands could
20397    /// never have touched. `gate` must recognise there is nothing to check
20398    /// before it ever asks for the lease.
20399    #[tokio::test]
20400    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
20401        crate::run::pin_test_home();
20402        let home = crate::run::home();
20403
20404        let tmp = tempfile::tempdir().expect("tempdir");
20405        let repo = tmp.path().join("repo");
20406        std::fs::create_dir_all(&repo).unwrap();
20407        init_repo(&repo);
20408        // Unique to this test, so holding its lease cannot collide with
20409        // another test sharing the same process-wide `home`.
20410        let cache_dir = tmp.path().join("target");
20411
20412        let mut config = Config::default();
20413        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
20414        // `verify.gate` stays empty (the default). Any accidental cache
20415        // acquisition must report contention immediately, regardless of how
20416        // long unrelated Git subprocesses take under parallel test load.
20417        config.graph.timeout_verify = Some(2);
20418        config.graph.timeout_resource_wait = 0;
20419
20420        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20421        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20422            .expect("no io error acquiring directly")
20423        {
20424            crate::cache::AcquireOutcome::Acquired(g) => g,
20425            crate::cache::AcquireOutcome::Busy(b) => {
20426                panic!("expected the direct acquire to win the lease first: {b:?}")
20427            }
20428        };
20429
20430        let mut state = RunState::new(
20431            repo.clone(),
20432            "main".to_owned(),
20433            "deadbeef".to_owned(),
20434            "task".to_owned(),
20435            config,
20436        );
20437        state.candidates = vec![Candidate {
20438            index: 0,
20439            label: 'A',
20440            agent: "alpha".to_owned(),
20441            branch: "does-not-exist".to_owned(),
20442            worktree: repo.clone(),
20443            summary: String::new(),
20444            stat: String::new(),
20445            files: 0,
20446            commits: 0,
20447            empty: false,
20448            failed: None,
20449            verified_noop: None,
20450            duration_ms: 0,
20451            folded: false,
20452        }];
20453        state.tally = Some(Tally {
20454            first_choice: BTreeMap::from([('A', 1)]),
20455            borda: BTreeMap::new(),
20456            winner: 'A',
20457            rankings: 1,
20458            unanimous_initial: true,
20459            deliberated: false,
20460            changed_votes: 0,
20461            unanimous_final: true,
20462            tie_break: None,
20463            judges: 0,
20464            present: 0,
20465            quorum: 0,
20466            met_quorum: true,
20467            uncontested: Some("only candidate A produced a change".to_owned()),
20468        });
20469        state.reviews = vec![ReviewRound {
20470            round: 1,
20471            head: "deadbeef".to_owned(),
20472            verified_head: None,
20473            verified_at: None,
20474            reviews: Vec::new(),
20475            e2e: Vec::new(),
20476            fix: None,
20477            blocking: 0,
20478            answered: 0,
20479            expected: 0,
20480            clean: true,
20481            verify_retried: false,
20482            e2e_deferred: false,
20483            e2e_defer_reason: None,
20484            progressed: false,
20485            vote_split: false,
20486            reconsideration: Vec::new(),
20487            verdict: None,
20488        }];
20489
20490        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20491        state.reviews[0].e2e = vec![CommandOutcome {
20492            command: state.config.verify.e2e[0].clone(),
20493            code: Some(0),
20494            output_tail: String::new(),
20495            duration_ms: 0,
20496            resource_blocked: false,
20497        }];
20498        let mut runner = Runner {
20499            state,
20500            roles: ResolvedRoles {
20501                implementers: Vec::new(),
20502                judges: Vec::new(),
20503                reviewers: Vec::new(),
20504                fixer: None,
20505                conductor: conductor(),
20506                implementer_roster: Vec::new(),
20507                judge_roster: Vec::new(),
20508                reviewer_roster: Vec::new(),
20509            },
20510            sem: Arc::new(Semaphore::new(1)),
20511            pause: Pause::new(),
20512            interrupt: Pause::new(),
20513        };
20514
20515        runner.gate().await.expect("gate");
20516        assert!(runner.state.resource_block.is_none());
20517        assert!(
20518            !runner
20519                .state
20520                .events
20521                .iter()
20522                .any(|event| { event.message.contains("waiting for the shared build cache") }),
20523            "a gate with nothing to run must never request the held cache lease"
20524        );
20525        assert!(
20526            runner.state.gate_ran,
20527            "zero commands is still a real, immediate attempt"
20528        );
20529        assert!(runner.state.gate.is_empty());
20530        assert_ne!(
20531            runner.state.status,
20532            RunStatus::Blocked,
20533            "must not read as resource-blocked on a lease it never asked for"
20534        );
20535    }
20536
20537    /// A separate process carries a live worker's environment throughout all
20538    /// verification entry points; the parallel harness never changes its env.
20539    #[cfg(unix)]
20540    #[tokio::test]
20541    async fn verification_is_standalone_inside_a_live_worker() {
20542        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20543        if std::env::var_os(CHILD).is_none() {
20544            let home = tempfile::tempdir().unwrap();
20545            let generation = crate::worker::pin_current(home.path()).unwrap();
20546            let output = std::process::Command::new(std::env::current_exe().unwrap())
20547                .args([
20548                    "--exact",
20549                    "graph::tests::verification_is_standalone_inside_a_live_worker",
20550                    "--nocapture",
20551                ])
20552                .env(CHILD, "1")
20553                .env("MAGI_HOME", home.path())
20554                .env("MAGI_WORKER_HOME", home.path())
20555                .env("MAGI_WORKER_ID", "abcdef")
20556                .env("MAGI_WORKER_GENERATION", &generation.id)
20557                .env("MAGI_WORKER_EXE", &generation.executable)
20558                .env("MAGI_RUN", "verification-parent")
20559                .env("MAGI_NODE", "gate")
20560                .env("MAGI_SEAT", "worker-fixture")
20561                .env("MAGI_PROCESS_OWNER", "fixture-owner")
20562                .env("MAGI_NO_AUTOUPDATE", "0")
20563                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20564                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20565                .env(
20566                    "VERIFY_EXPECT_TOOLCHAIN",
20567                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20568                )
20569                .env(
20570                    "VERIFY_EXPECT_CACHE",
20571                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20572                )
20573                .output()
20574                .unwrap();
20575            assert!(
20576                output.status.success()
20577                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20578                "{}\n{}",
20579                String::from_utf8_lossy(&output.stdout),
20580                String::from_utf8_lossy(&output.stderr)
20581            );
20582            return;
20583        }
20584        let home = crate::run::home();
20585        let directory = home.join("workers/jobs/abcdef");
20586        std::fs::create_dir_all(&directory).unwrap();
20587        let job = crate::worker::Job {
20588            id: "abcdef".into(),
20589            sequence: 1,
20590            kind: "task".into(),
20591            key: "task:verification".into(),
20592            payload: serde_json::json!({}),
20593            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20594            version: Some(env!("CARGO_PKG_VERSION").into()),
20595            status: crate::worker::Status::Running,
20596            result: None,
20597            pid: Some(std::process::id()),
20598            created_at: 0,
20599            finished_at: None,
20600        };
20601        std::fs::write(
20602            directory.join("job.json"),
20603            serde_json::to_vec(&job).unwrap(),
20604        )
20605        .unwrap();
20606        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20607        let assert_worker = || {
20608            assert_eq!(
20609                crate::worker::current_job(&home).unwrap().unwrap().id,
20610                job.id
20611            );
20612            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20613            assert_eq!(
20614                crate::worker::helper_executable(&home).unwrap(),
20615                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20616            );
20617            assert!(
20618                crate::updater::Checker::new(&crate::config::Update {
20619                    mode: crate::config::UpdateMode::Notify,
20620                    interval: None,
20621                    auto_restart: false,
20622                })
20623                .is_none(),
20624                "the driver remains pinned, even while tests are standalone"
20625            );
20626        };
20627        assert_worker();
20628        let repo = home.join("repo");
20629        std::fs::create_dir_all(&repo).unwrap();
20630        init_repo(&repo);
20631        // Keep storage, seat/control context, containment and tool lookup.
20632        // No inherited values (or credentials) are printed on success.
20633        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20634            test -z "${MAGI_WORKER_GENERATION+x}" &&
20635            test -z "${MAGI_WORKER_HOME+x}" &&
20636            test -z "${MAGI_WORKER_EXE+x}" &&
20637            test "$MAGI_HOME/repo" -ef . &&
20638            test "$MAGI_RUN" = verification-parent &&
20639            test "$MAGI_NODE" = gate &&
20640            test "$MAGI_SEAT" = worker-fixture &&
20641            test -n "$MAGI_PROCESS_OWNER" &&
20642            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20643            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20644            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20645            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20646        // Negative control: an ordinary child inherits the contamination.
20647        assert!(
20648            !std::process::Command::new("sh")
20649                .args(["-c", probe])
20650                .current_dir(&repo)
20651                .status()
20652                .unwrap()
20653                .success()
20654        );
20655        let unrelated_home = tempfile::tempdir().unwrap();
20656        assert!(
20657            crate::worker::current_job(unrelated_home.path()).is_err(),
20658            "an inherited worker id cannot be resolved in a unit fixture's home"
20659        );
20660        let mut runner = runner_at(RunStatus::Gating);
20661        runner.state.repo = repo.clone();
20662        runner.state.config.disk.min_free_bytes = 0;
20663        runner.state.config.coordination = crate::config::test_coordination();
20664        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20665        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20666        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20667        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20668        let winner = Candidate {
20669            index: 0,
20670            label: 'A',
20671            agent: "fixture".into(),
20672            branch: "main".into(),
20673            worktree: repo.clone(),
20674            summary: String::new(),
20675            stat: String::new(),
20676            files: 0,
20677            commits: 0,
20678            empty: false,
20679            failed: None,
20680            verified_noop: None,
20681            duration_ms: 0,
20682            folded: false,
20683        };
20684        let (e2e, retried, pids) = run_e2e_with_retry(
20685            &mut runner.state,
20686            &["sh".into(), "-c".into()],
20687            &[format!("{probe} && printf initial")],
20688            &repo,
20689            Duration::from_secs(10),
20690            "initial",
20691        )
20692        .await;
20693        assert!(!retried && pids.is_empty());
20694        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20695        assert_eq!(e2e[0].output_tail, "initial");
20696        let gate = runner.run_gate(&winner).await.unwrap();
20697        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20698        // An interrupted repair must recheck e2e from saved state. Landing
20699        // reuses that state and must run both pre-gate and gate again.
20700        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20701            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20702            passed: false,
20703            outcomes: Vec::new(),
20704        });
20705        runner.state.save().unwrap();
20706        let mut resumed = RunState::load(&runner.state.id).unwrap();
20707        let wait = crate::repo_lock::WaitPolicy::default();
20708        assert!(matches!(
20709            check_gate_repair_e2e(&mut resumed, &winner, &wait)
20710                .await
20711                .unwrap(),
20712            GateFix::Retry
20713        ));
20714        let saved = RunState::load(&resumed.id).unwrap();
20715        let repair = saved.gate_repair_e2e.unwrap();
20716        assert!(repair.passed);
20717        assert_eq!(repair.outcomes.len(), 1);
20718        assert!(repair.outcomes[0].ok());
20719        let landing = run_landing_gate(&mut resumed, &winner, &wait)
20720            .await
20721            .unwrap();
20722        assert_eq!(landing.len(), 1);
20723        assert!(landing[0].ok(), "{landing:?}");
20724        assert_eq!(
20725            std::fs::read_to_string(repo.join("phases")).unwrap(),
20726            "pre\ngate\ne2e\npre\ngate\n"
20727        );
20728        assert_worker();
20729    }
20730
20731    #[cfg(unix)]
20732    #[tokio::test]
20733    async fn verification_keeps_the_panic_and_the_full_output() {
20734        crate::run::pin_test_home();
20735        let tmp = tempfile::tempdir().unwrap();
20736        let mut config = Config::default();
20737        config.disk.min_free_bytes = 0;
20738        let mut state = RunState::new(
20739            tmp.path().to_owned(),
20740            "main".into(),
20741            "base".into(),
20742            "verification evidence".into(),
20743            config,
20744        );
20745        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20746                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
20747                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20748        let mut names = Vec::new();
20749        for _ in 0..2 {
20750            let (outcomes, _) = run_commands(
20751                &mut state,
20752                "gate",
20753                "gate",
20754                0,
20755                &["sh".into(), "-c".into()],
20756                &[script.into()],
20757                tmp.path(),
20758                Duration::from_secs(60),
20759            )
20760            .await;
20761            let tail = &outcomes[0].output_tail;
20762            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20763            assert!(tail.contains("boom assertion"));
20764            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20765            let line = tail.lines().next().unwrap();
20766            names.push(line.to_owned());
20767        }
20768        assert_ne!(names[0], names[1], "the second run must not overwrite");
20769        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20770        assert!(full.len() > 500_000 && full.contains("boom assertion"));
20771        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20772
20773        // Storage failures are evidence too; preserve the command's verdict.
20774        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20775        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20776        let (outcomes, _) = run_commands(
20777            &mut state,
20778            "gate",
20779            "gate",
20780            0,
20781            &["sh".into(), "-c".into()],
20782            &["echo failure-output; exit 7".into()],
20783            tmp.path(),
20784            Duration::from_secs(10),
20785        )
20786        .await;
20787        assert_eq!(outcomes[0].code, Some(7));
20788        assert!(state.events.iter().any(|e| {
20789            e.node == "gate" && e.message.contains("could not save the full gate output:")
20790        }));
20791    }
20792
20793    #[cfg(unix)]
20794    #[tokio::test]
20795    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20796        crate::run::pin_test_home();
20797        let tmp = tempfile::tempdir().unwrap();
20798        let mut config = Config::default();
20799        config.disk.min_free_bytes = 0;
20800        let mut state = RunState::new(
20801            tmp.path().to_owned(),
20802            "main".into(),
20803            "base".into(),
20804            "verification evidence".into(),
20805            config,
20806        );
20807        let (outcomes, pids) = run_commands(
20808            &mut state,
20809            "gate",
20810            "gate",
20811            1,
20812            &["sh".into(), "-c".into()],
20813            &[
20814                "printf before-timeout; exec sleep 30".into(),
20815                "kill -TERM $$".into(),
20816                "exit 7".into(),
20817                "printf success".into(),
20818            ],
20819            tmp.path(),
20820            Duration::from_secs(1),
20821        )
20822        .await;
20823        wait_for_timed_out_children_to_die(&pids).await;
20824        assert_eq!(
20825            pids.len(),
20826            1,
20827            "unleased timeout retains its death-check PID"
20828        );
20829        assert_eq!(outcomes.len(), 4);
20830        assert_eq!(outcomes[0].code, None);
20831        assert!(outcomes[0].output_tail.contains("before-timeout"));
20832        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20833        assert_eq!(outcomes[1].code, None);
20834        assert!(outcomes[1].output_tail.contains("command terminated:"));
20835        assert!(!outcomes[1].output_tail.contains("timed out"));
20836        assert_eq!(outcomes[2].code, Some(7));
20837        assert!(!outcomes[2].output_tail.contains("timed out"));
20838        assert_eq!(outcomes[3].code, Some(0));
20839        assert_eq!(outcomes[3].output_tail, "success");
20840        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20841    }
20842
20843    /// The addendum's second gap: a `verify.gate` command running for real
20844    /// wall-clock time had nothing at all to show for it in `active` before
20845    /// `run_commands` learned to record it — a run could sit in `Gating` for
20846    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
20847    /// what was actually happening. Proven with a genuinely still-running
20848    /// command, not just a before/after check on the final state: a poller
20849    /// task reads the same `run.json` `gate()` is writing, the same way the
20850    /// phone or `magi show` would, while the shell command is still blocked
20851    /// on its own release marker.
20852    #[tokio::test]
20853    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20854        crate::run::pin_test_home();
20855
20856        let tmp = tempfile::tempdir().expect("tempdir");
20857        let repo = tmp.path().join("repo");
20858        std::fs::create_dir_all(&repo).unwrap();
20859        init_repo(&repo);
20860
20861        let mut config = Config::default();
20862        config.disk.min_free_bytes = 0;
20863        config.verify.gate = vec![
20864            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20865             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20866                .to_owned(),
20867        ];
20868
20869        let mut state = RunState::new(
20870            repo.clone(),
20871            "main".to_owned(),
20872            "deadbeef".to_owned(),
20873            "task".to_owned(),
20874            config,
20875        );
20876        let run_id = state.id.clone();
20877        state.candidates = vec![Candidate {
20878            index: 0,
20879            label: 'A',
20880            agent: "alpha".to_owned(),
20881            branch: "does-not-exist".to_owned(),
20882            worktree: repo.clone(),
20883            summary: String::new(),
20884            stat: String::new(),
20885            files: 0,
20886            commits: 0,
20887            empty: false,
20888            failed: None,
20889            verified_noop: None,
20890            duration_ms: 0,
20891            folded: false,
20892        }];
20893        state.tally = Some(Tally {
20894            first_choice: BTreeMap::from([('A', 1)]),
20895            borda: BTreeMap::new(),
20896            winner: 'A',
20897            rankings: 1,
20898            unanimous_initial: true,
20899            deliberated: false,
20900            changed_votes: 0,
20901            unanimous_final: true,
20902            tie_break: None,
20903            judges: 0,
20904            present: 0,
20905            quorum: 0,
20906            met_quorum: true,
20907            uncontested: Some("only candidate A produced a change".to_owned()),
20908        });
20909        state.reviews = vec![ReviewRound {
20910            round: 1,
20911            head: "deadbeef".to_owned(),
20912            verified_head: None,
20913            verified_at: None,
20914            reviews: Vec::new(),
20915            e2e: Vec::new(),
20916            fix: None,
20917            blocking: 0,
20918            answered: 0,
20919            expected: 0,
20920            clean: true,
20921            verify_retried: false,
20922            e2e_deferred: false,
20923            e2e_defer_reason: None,
20924            progressed: false,
20925            vote_split: false,
20926            reconsideration: Vec::new(),
20927            verdict: None,
20928        }];
20929
20930        let mut runner = Runner {
20931            state,
20932            roles: ResolvedRoles {
20933                implementers: Vec::new(),
20934                judges: Vec::new(),
20935                reviewers: Vec::new(),
20936                fixer: None,
20937                conductor: conductor(),
20938                implementer_roster: Vec::new(),
20939                judge_roster: Vec::new(),
20940                reviewer_roster: Vec::new(),
20941            },
20942            sem: Arc::new(Semaphore::new(1)),
20943            pause: Pause::new(),
20944            interrupt: Pause::new(),
20945        };
20946
20947        let started_marker = repo.join("started.marker");
20948        let release_marker = repo.join("release.marker");
20949        let poller = tokio::spawn(async move {
20950            // Bounded so a regression that never records the task entry
20951            // fails this test in seconds instead of hanging the suite —
20952            // the same shape `a_park_requested_while_a_seat_is_mid_call_
20953            // does_not_cut_it_short` uses for the same reason.
20954            for _ in 0..100 {
20955                if started_marker.exists()
20956                    && let Ok(s) = crate::run::RunState::load(&run_id)
20957                    && let Some(a) = s.active.get("gate")
20958                {
20959                    std::fs::write(&release_marker, b"go").expect("release marker");
20960                    return Some(a.clone());
20961                }
20962                tokio::time::sleep(Duration::from_millis(50)).await;
20963            }
20964            None
20965        });
20966
20967        runner.gate().await.expect("gate");
20968        let captured = poller.await.expect("poller task");
20969        let captured = captured.expect(
20970            "the poller never saw a `gate` task entry in run.json while the command was \
20971             still blocked on its own release marker",
20972        );
20973
20974        assert_eq!(captured.task.as_deref(), Some("gate"));
20975        assert_eq!(captured.node, "gate");
20976        assert_eq!(captured.index, Some(1));
20977        assert_eq!(captured.total, Some(1));
20978        assert!(
20979            captured
20980                .command
20981                .as_deref()
20982                .is_some_and(|c| c.contains("started.marker")),
20983            "{captured:?}"
20984        );
20985
20986        assert!(
20987            runner.state.active.is_empty(),
20988            "the entry must be cleared once the command actually finished: {:?}",
20989            runner.state.active
20990        );
20991        assert!(runner.state.gate_ran);
20992        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20993    }
20994
20995    /// The hand-off over a blocking finding a reviewer rejected on leaves a
20996    /// record for `land`; one with only a Minor, or no reject, leaves none.
20997    #[tokio::test]
20998    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20999        use crate::verdict::{Finding, ReviewVote, Severity};
21000        crate::run::pin_test_home();
21001        let tmp = tempfile::tempdir().expect("tempdir");
21002        let repo = tmp.path().join("repo");
21003        std::fs::create_dir_all(&repo).unwrap();
21004        init_repo(&repo);
21005
21006        for (severity, vote, expect) in [
21007            (Severity::Major, ReviewVote::Reject, true),
21008            (Severity::Minor, ReviewVote::Reject, false),
21009            (Severity::Major, ReviewVote::Approve, false),
21010        ] {
21011            let mut round = review_round(false, 1, 1, 1, false, true);
21012            round.reviews = vec![ReviewRecord {
21013                reviewer: 1,
21014                agent: "alpha".to_owned(),
21015                summary: String::new(),
21016                findings: vec![Finding {
21017                    id: "R1-1-1".to_owned(),
21018                    severity,
21019                    file: None,
21020                    line: None,
21021                    title: "t".to_owned(),
21022                    detail: String::new(),
21023                }],
21024                vote: Some(vote),
21025                failed: None,
21026                duration_ms: 0,
21027                attempts: 0,
21028            }];
21029            let mut state = RunState::new(
21030                repo.clone(),
21031                "main".to_owned(),
21032                "deadbeef".to_owned(),
21033                "task".to_owned(),
21034                Config::default(),
21035            );
21036            state.reviews = vec![round];
21037            let mut runner = Runner {
21038                state,
21039                roles: ResolvedRoles {
21040                    implementers: Vec::new(),
21041                    judges: Vec::new(),
21042                    reviewers: Vec::new(),
21043                    fixer: None,
21044                    conductor: conductor(),
21045                    implementer_roster: Vec::new(),
21046                    judge_roster: Vec::new(),
21047                    reviewer_roster: Vec::new(),
21048                },
21049                sem: Arc::new(Semaphore::new(1)),
21050                pause: Pause::new(),
21051                interrupt: Pause::new(),
21052            };
21053            let shell = runner.state.config.shell();
21054            runner
21055                .stop_reviewing("round budget spent", &shell, &repo)
21056                .await
21057                .expect("stop_reviewing");
21058            assert_eq!(runner.state.status, RunStatus::Gating);
21059            assert_eq!(
21060                runner.state.contested_handoff.is_some(),
21061                expect,
21062                "{severity:?} + {vote:?}"
21063            );
21064        }
21065    }
21066
21067    /// The shape the incident this whole fix responds to actually had: the
21068    /// round budget spent, the last round's own e2e blocked on the shared
21069    /// build cache (held here by a live pid — this test process — exactly
21070    /// `cache`'s own unit tests' pattern for "another owner, still alive"
21071    /// without forking a process). `stop_reviewing` must retry it — not
21072    /// silently leave the round looking untouched (the catch-up-only half of
21073    /// the bug), and not read the contention as a red `e2e` and block the
21074    /// run on it (the other half). Called directly, the same way
21075    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
21076    /// above exercises `gate`, so this never needs a real cargo build to
21077    /// reach: the lease is never released, so `with_cache_lease` never gets
21078    /// past acquiring it into anything that would need a real workspace.
21079    #[tokio::test]
21080    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
21081        crate::run::pin_test_home();
21082        let home = crate::run::home();
21083
21084        let tmp = tempfile::tempdir().expect("tempdir");
21085        let repo = tmp.path().join("repo");
21086        std::fs::create_dir_all(&repo).unwrap();
21087        init_repo(&repo);
21088        let head = crate::git::rev_parse(&repo, "HEAD")
21089            .await
21090            .expect("rev-parse");
21091        // Unique to this test, so holding its lease cannot collide with
21092        // another test sharing the same process-wide `home`.
21093        let cache_dir = tmp.path().join("target");
21094
21095        let mut config = Config::default();
21096        config.verify.e2e = vec![format!(
21097            "CARGO_TARGET_DIR='{}' test -f README.md",
21098            cache_dir.display()
21099        )];
21100        config.graph.review_rounds = 1;
21101        // Bounded so a regression that does start waiting fails the test in
21102        // seconds, not hangs it.
21103        config.graph.timeout_verify = Some(2);
21104        config.graph.timeout_resource_wait = 2;
21105
21106        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21107        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21108            .expect("no io error acquiring directly")
21109        {
21110            crate::cache::AcquireOutcome::Acquired(g) => g,
21111            crate::cache::AcquireOutcome::Busy(b) => {
21112                panic!("expected the direct acquire to win the lease first: {b:?}")
21113            }
21114        };
21115
21116        let mut state = RunState::new(
21117            repo.clone(),
21118            "main".to_owned(),
21119            head.clone(),
21120            "task".to_owned(),
21121            config,
21122        );
21123        state.candidates = vec![Candidate {
21124            index: 0,
21125            label: 'A',
21126            agent: "alpha".to_owned(),
21127            branch: "does-not-exist".to_owned(),
21128            worktree: repo.clone(),
21129            summary: String::new(),
21130            stat: String::new(),
21131            files: 0,
21132            commits: 0,
21133            empty: false,
21134            failed: None,
21135            verified_noop: None,
21136            duration_ms: 0,
21137            folded: false,
21138        }];
21139        state.tally = Some(Tally {
21140            first_choice: BTreeMap::from([('A', 1)]),
21141            borda: BTreeMap::new(),
21142            winner: 'A',
21143            rankings: 1,
21144            unanimous_initial: true,
21145            deliberated: false,
21146            changed_votes: 0,
21147            unanimous_final: true,
21148            tie_break: None,
21149            judges: 0,
21150            present: 0,
21151            quorum: 0,
21152            met_quorum: true,
21153            uncontested: Some("only candidate A produced a change".to_owned()),
21154        });
21155        // The round budget's last round, deferred: `needs_catchup_run`'s
21156        // other trigger. `stop_reviewing`'s retry machinery must treat this
21157        // exactly like a resource-blocked attempt once it actually runs.
21158        state.reviews = vec![ReviewRound {
21159            round: 1,
21160            head: head.clone(),
21161            verified_head: None,
21162            verified_at: None,
21163            reviews: Vec::new(),
21164            e2e: Vec::new(),
21165            fix: None,
21166            blocking: 1,
21167            answered: 1,
21168            expected: 1,
21169            clean: false,
21170            verify_retried: false,
21171            e2e_deferred: true,
21172            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
21173            progressed: false,
21174            vote_split: false,
21175            reconsideration: Vec::new(),
21176            verdict: None,
21177        }];
21178
21179        let mut runner = Runner {
21180            state,
21181            roles: ResolvedRoles {
21182                implementers: Vec::new(),
21183                judges: Vec::new(),
21184                reviewers: Vec::new(),
21185                fixer: None,
21186                conductor: conductor(),
21187                implementer_roster: Vec::new(),
21188                judge_roster: Vec::new(),
21189                reviewer_roster: Vec::new(),
21190            },
21191            sem: Arc::new(Semaphore::new(1)),
21192            pause: Pause::new(),
21193            interrupt: Pause::new(),
21194        };
21195
21196        let shell = runner.state.config.shell();
21197        runner
21198            .stop_reviewing("round budget spent", &shell, &repo)
21199            .await
21200            .expect("stop_reviewing");
21201
21202        let last = runner.state.reviews.last().expect("round record");
21203        assert_eq!(
21204            last.e2e_status(),
21205            E2eStatus::ResourceBlocked,
21206            "the shared cache is still held; the attempt must read as blocked, not deferred or \
21207             failed: {last:?}"
21208        );
21209        assert_eq!(
21210            last.verified_head.as_deref(),
21211            Some(head.as_str()),
21212            "which commit this attempt targeted is known even though nothing finished checking \
21213             it"
21214        );
21215        let first_attempt_at = last
21216            .verified_at
21217            .expect("when this attempt ran is known too");
21218        assert_ne!(
21219            runner.state.status,
21220            RunStatus::Blocked,
21221            "contention is evidence about the machine, not the patch — it must not settle the \
21222             run as blocked: {:?}",
21223            runner.state.status
21224        );
21225        assert!(
21226            !runner
21227                .state
21228                .events
21229                .iter()
21230                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
21231            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
21232            runner.state.events
21233        );
21234
21235        // The cache is still held: a later reentry must retry the same
21236        // round's verification again — not leave it looking exactly as
21237        // untouched as the first blocked attempt, which is indistinguishable
21238        // from never having tried again at all.
21239        runner
21240            .stop_reviewing("round budget spent", &shell, &repo)
21241            .await
21242            .expect("stop_reviewing retry");
21243        assert_eq!(
21244            runner.state.reviews.len(),
21245            1,
21246            "no new round was started: {:?}",
21247            runner.state.reviews
21248        );
21249        let last = runner.state.reviews.last().expect("round record");
21250        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
21251        assert!(
21252            last.verified_at.expect("still known") > first_attempt_at,
21253            "a second reentry must be a fresh attempt, not a stale copy of the first"
21254        );
21255        assert_ne!(runner.state.status, RunStatus::Blocked);
21256
21257        held.release();
21258    }
21259
21260    #[cfg(unix)]
21261    #[tokio::test]
21262    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
21263        ask_test_home();
21264        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
21265            let tmp = tempfile::tempdir().unwrap();
21266            let repo = tmp.path().join("repo");
21267            std::fs::create_dir(&repo).unwrap();
21268            init_repo(&repo);
21269            let run_git = |args: &[&str]| {
21270                let out = std::process::Command::new("git")
21271                    .args(args)
21272                    .current_dir(&repo)
21273                    .output()
21274                    .unwrap();
21275                assert!(
21276                    out.status.success(),
21277                    "{args:?}: {}",
21278                    String::from_utf8_lossy(&out.stderr)
21279                );
21280            };
21281            let origin = tmp.path().join("origin.git");
21282            run_git(&["init", "--bare", origin.to_str().unwrap()]);
21283            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
21284            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
21285            run_git(&["add", "spec.rs"]);
21286            run_git(&["commit", "-m", "initial spec"]);
21287            run_git(&["push", "origin", "main"]);
21288            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
21289            run_git(&["checkout", "-b", "fix/review"]);
21290            std::fs::write(
21291                repo.join("check.rs"),
21292                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
21293            )
21294            .unwrap();
21295            run_git(&["add", "check.rs"]);
21296            run_git(&["commit", "-m", "use spec"]);
21297            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
21298            run_git(&["checkout", "main"]);
21299            std::fs::write(
21300                repo.join("spec.rs"),
21301                "struct AgentSpec { max_turn_secs: u64 }\n",
21302            )
21303            .unwrap();
21304            run_git(&["add", "spec.rs"]);
21305            run_git(&["commit", "-m", "extend spec"]);
21306            run_git(&["push", "origin", "main"]);
21307            run_git(&["checkout", "fix/review"]);
21308
21309            let mut runner = runner_at(RunStatus::Reviewing);
21310            runner.state.repo = repo.clone();
21311            runner.state.base_commit = base;
21312            runner.state.pr_head = Some("fix/review".into());
21313            runner.state.config.graph.review_rounds = max_rounds;
21314            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
21315            runner.state.config.graph.sessions = false;
21316            runner.state.config.disk.min_free_bytes = 0;
21317            runner.state.config.verify.e2e =
21318                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
21319            runner.state.candidates = vec![Candidate {
21320                index: 0,
21321                label: 'A',
21322                agent: "fixture".into(),
21323                branch: "fix/review".into(),
21324                worktree: repo.clone(),
21325                summary: String::new(),
21326                stat: String::new(),
21327                files: 1,
21328                commits: 1,
21329                empty: false,
21330                failed: None,
21331                verified_noop: None,
21332                duration_ms: 0,
21333                folded: false,
21334            }];
21335            runner.state.tally = Some(Tally {
21336                first_choice: BTreeMap::new(),
21337                borda: BTreeMap::new(),
21338                winner: 'A',
21339                rankings: 0,
21340                unanimous_initial: true,
21341                deliberated: false,
21342                changed_votes: 0,
21343                unanimous_final: true,
21344                tie_break: None,
21345                judges: 0,
21346                present: 0,
21347                quorum: 0,
21348                met_quorum: true,
21349                uncontested: Some("review-only".into()),
21350            });
21351            let mut last = review_round(false, 0, 1, 1, false, false);
21352            last.head = reviewed.clone();
21353            last.e2e_deferred = true;
21354            last.e2e[0].resource_blocked = true;
21355            last.e2e[0].output_tail = "waiting for the shared build cache".into();
21356            if max_rounds == 1 {
21357                last.e2e.clear();
21358            }
21359            runner.state.reviews = vec![last];
21360            let calls = tmp.path().join("fix-calls");
21361            let mut fixer = spec("repair");
21362            fixer
21363                .env
21364                .insert("FIX_CALLS".into(), calls.display().to_string());
21365            fixer.command = vec!["sh".into(), "-c".into(), r#"
21366                echo fix >> "$FIX_CALLS"
21367                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
21368                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
21369                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
21370                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
21371            "#.into()];
21372            let mut reviewer = spec("reviewer");
21373            reviewer.command = vec![
21374                "sh".into(),
21375                "-c".into(),
21376                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
21377                    .into(),
21378            ];
21379            runner.roles.fixer = Some(vec![fixer]);
21380            runner.roles.reviewers = vec![reviewer];
21381
21382            runner.sync_to_base().await.unwrap();
21383            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
21384            assert_ne!(rebased, reviewed);
21385            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
21386            if reentry {
21387                // Simulate a crash after catch-up evidence was saved but before
21388                // dispatching the fixer. No dedicated pending-repair field exists.
21389                let shell = runner.state.config.shell();
21390                runner
21391                    .stop_unresolved_review("deferred verification", &shell, &repo)
21392                    .await
21393                    .unwrap();
21394                assert!(runner.catchup_repairable());
21395                assert!(!calls.exists());
21396                runner.state = RunState::load(&runner.state.id).unwrap();
21397                assert!(runner.catchup_repairable());
21398            }
21399            runner.review_loop().await.unwrap();
21400            let first = &runner.state.reviews[0];
21401            assert_eq!(first.head, reviewed);
21402            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
21403            assert!(
21404                first.verify_retried,
21405                "a reproduced build failure gets exactly one retry: {:?}",
21406                first.e2e
21407            );
21408            assert!(first.e2e[0].output_tail.contains("E0063"));
21409            assert!(!runner.catchup_repairable());
21410            if max_rounds == 2 {
21411                assert_eq!(
21412                    runner.state.status,
21413                    RunStatus::Gating,
21414                    "{}",
21415                    crate::report::run(&runner.state)
21416                );
21417                assert_eq!(runner.state.reviews.len(), 2);
21418                assert!(first.fix.as_ref().unwrap().committed);
21419                assert!(runner.state.reviews[1].clean);
21420                assert_eq!(
21421                    runner
21422                        .state
21423                        .reviews
21424                        .iter()
21425                        .filter(|r| r.fix.is_some())
21426                        .count(),
21427                    1
21428                );
21429                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
21430                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
21431            } else {
21432                assert_eq!(runner.state.status, RunStatus::Blocked);
21433                assert!(first.fix.is_none());
21434                assert!(!calls.exists());
21435                let recovery = crate::recovery::assess(
21436                    "prior",
21437                    Some(&runner.state),
21438                    &runner.landing_base(),
21439                    crate::recovery::Relation::Diverged,
21440                );
21441                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21442                let reasons = recovery.reasons.join("\n");
21443                assert!(
21444                    reasons.contains(&reviewed) && reasons.contains(&rebased),
21445                    "{reasons}"
21446                );
21447                assert!(
21448                    reasons.contains("base sync to")
21449                        && reasons.contains("additional review is required"),
21450                    "{reasons}"
21451                );
21452            }
21453        }
21454    }
21455
21456    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
21457    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
21458    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
21459    /// own top-of-function fast path (`review_conclusion`) correctly reads
21460    /// this shape as `None` rather than guessing `Blocked`, and the loop's
21461    /// own `for` range is empty once the round budget is spent, so
21462    /// `review_loop` must retry the check itself rather than silently doing
21463    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
21464    /// above exercises directly, but through `review_loop`'s own entry point
21465    /// this time, proving the wiring between the two rather than just the
21466    /// retry logic in isolation.
21467    #[tokio::test]
21468    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21469        crate::run::pin_test_home();
21470        let home = crate::run::home();
21471
21472        let tmp = tempfile::tempdir().expect("tempdir");
21473        let repo = tmp.path().join("repo");
21474        std::fs::create_dir_all(&repo).unwrap();
21475        init_repo(&repo);
21476        let head = crate::git::rev_parse(&repo, "HEAD")
21477            .await
21478            .expect("rev-parse");
21479        let cache_dir = tmp.path().join("target");
21480
21481        let mut config = Config::default();
21482        config.verify.e2e = vec![format!(
21483            "CARGO_TARGET_DIR='{}' test -f README.md",
21484            cache_dir.display()
21485        )];
21486        config.graph.review_rounds = 1;
21487        config.graph.timeout_verify = Some(2);
21488        config.graph.timeout_resource_wait = 2;
21489
21490        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21491        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21492            .expect("no io error acquiring directly")
21493        {
21494            crate::cache::AcquireOutcome::Acquired(g) => g,
21495            crate::cache::AcquireOutcome::Busy(b) => {
21496                panic!("expected the direct acquire to win the lease first: {b:?}")
21497            }
21498        };
21499
21500        let mut state = RunState::new(
21501            repo.clone(),
21502            "main".to_owned(),
21503            head.clone(),
21504            "task".to_owned(),
21505            config,
21506        );
21507        state.candidates = vec![Candidate {
21508            index: 0,
21509            label: 'A',
21510            agent: "alpha".to_owned(),
21511            branch: "does-not-exist".to_owned(),
21512            worktree: repo.clone(),
21513            summary: String::new(),
21514            stat: String::new(),
21515            files: 0,
21516            commits: 0,
21517            empty: false,
21518            failed: None,
21519            verified_noop: None,
21520            duration_ms: 0,
21521            folded: false,
21522        }];
21523        state.tally = Some(Tally {
21524            first_choice: BTreeMap::from([('A', 1)]),
21525            borda: BTreeMap::new(),
21526            winner: 'A',
21527            rankings: 1,
21528            unanimous_initial: true,
21529            deliberated: false,
21530            changed_votes: 0,
21531            unanimous_final: true,
21532            tie_break: None,
21533            judges: 0,
21534            present: 0,
21535            quorum: 0,
21536            met_quorum: true,
21537            uncontested: Some("only candidate A produced a change".to_owned()),
21538        });
21539        // The exact shape a prior process's `stop_reviewing` would have left
21540        // on disk: the round budget's last round, a real attempt already
21541        // made and already resource-blocked.
21542        state.reviews = vec![ReviewRound {
21543            round: 1,
21544            head: head.clone(),
21545            verified_head: Some(head.clone()),
21546            verified_at: Some(jiff::Timestamp::now()),
21547            reviews: Vec::new(),
21548            e2e: vec![CommandOutcome {
21549                command: format!(
21550                    "CARGO_TARGET_DIR='{}' test -f README.md",
21551                    cache_dir.display()
21552                ),
21553                code: None,
21554                output_tail: "waiting for the shared build cache".to_owned(),
21555                duration_ms: 0,
21556                resource_blocked: true,
21557            }],
21558            fix: None,
21559            blocking: 1,
21560            answered: 1,
21561            expected: 1,
21562            clean: false,
21563            verify_retried: false,
21564            e2e_deferred: false,
21565            e2e_defer_reason: None,
21566            progressed: false,
21567            vote_split: false,
21568            reconsideration: Vec::new(),
21569            verdict: None,
21570        }];
21571
21572        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21573        let mut runner = Runner {
21574            state,
21575            roles: ResolvedRoles {
21576                implementers: Vec::new(),
21577                judges: Vec::new(),
21578                reviewers: Vec::new(),
21579                fixer: None,
21580                conductor: conductor(),
21581                implementer_roster: Vec::new(),
21582                judge_roster: Vec::new(),
21583                reviewer_roster: Vec::new(),
21584            },
21585            sem: Arc::new(Semaphore::new(1)),
21586            pause: Pause::new(),
21587            interrupt: Pause::new(),
21588        };
21589
21590        // The lease is still held throughout, so this reentry's own retry is
21591        // also contended — proving `review_loop` actually tried again (not
21592        // that it happened to succeed) is what the timestamp comparison
21593        // below is for.
21594        runner.review_loop().await.expect("review_loop");
21595
21596        assert_eq!(
21597            runner.state.reviews.len(),
21598            1,
21599            "no new round was started on top of the unresolved one: {:?}",
21600            runner.state.reviews
21601        );
21602        let last = &runner.state.reviews[0];
21603        assert_eq!(
21604            last.e2e_status(),
21605            E2eStatus::ResourceBlocked,
21606            "still contended: {last:?}"
21607        );
21608        assert!(
21609            last.verified_at.expect("still known") > first_attempt_at,
21610            "review_loop must have actually retried the check, not left it exactly as found"
21611        );
21612        assert_ne!(
21613            runner.state.status,
21614            RunStatus::Blocked,
21615            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21616            runner.state.status
21617        );
21618
21619        held.release();
21620    }
21621
21622    #[tokio::test]
21623    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21624        crate::run::pin_test_home();
21625        for status in [RunStatus::Blocked, RunStatus::Gating] {
21626            let mut runner = runner_at(status);
21627            runner.state.config.graph.land = true;
21628            runner.state.gate_ran = true;
21629            runner.state.merge = Some(MergeOutcome {
21630                mode: MergeMode::Pr,
21631                ok: false,
21632                detail: "the fixer produced no commit".into(),
21633                empty: false,
21634            });
21635            let url = "https://example.invalid/o/r/pull/1";
21636            runner.state.pr = Some(crate::run::PrRecord {
21637                url: url.into(),
21638                number: 1,
21639                state: "open".into(),
21640                checks: "red".into(),
21641                round: 1,
21642                rounds: 1,
21643                red_at_merge: Vec::new(),
21644            });
21645            runner.state.save().unwrap();
21646            // The nonexistent repository makes observation fail before a forge
21647            // command can run. We still must reach land, rather than silently
21648            // return from the graph with its old gate and merge records.
21649            runner.execute_graph().await.unwrap();
21650            assert_eq!(runner.state.status, RunStatus::Blocked);
21651            assert!(runner.state.land_sync_pending);
21652            assert!(runner.state.events.iter().any(|event|
21653                event.node == "land" && event.message == format!("watching {url}")));
21654            assert_eq!(
21655                runner.state.merge.as_ref().unwrap().detail,
21656                "the fixer produced no commit"
21657            );
21658        }
21659    }
21660
21661    #[tokio::test]
21662    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21663        crate::run::pin_test_home();
21664        let tmp = tempfile::tempdir().expect("tempdir");
21665        let repo = tmp.path().join("repo");
21666        std::fs::create_dir_all(&repo).unwrap();
21667        init_repo(&repo);
21668
21669        let mut config = Config::default();
21670        config.merge.mode = MergeMode::Pr;
21671        config.graph.land = true;
21672        config.graph.land_approval = false;
21673
21674        let mut state = RunState::new(
21675            repo.clone(),
21676            "main".to_owned(),
21677            "deadbeef".to_owned(),
21678            "task".to_owned(),
21679            config,
21680        );
21681        state.candidates = vec![Candidate {
21682            index: 0,
21683            label: 'A',
21684            agent: "alpha".to_owned(),
21685            branch: "does-not-exist".to_owned(),
21686            worktree: repo.clone(),
21687            summary: String::new(),
21688            stat: String::new(),
21689            files: 0,
21690            commits: 0,
21691            empty: false,
21692            failed: None,
21693            verified_noop: None,
21694            duration_ms: 0,
21695            folded: false,
21696        }];
21697        state.tally = Some(Tally {
21698            first_choice: BTreeMap::from([('A', 1)]),
21699            borda: BTreeMap::new(),
21700            winner: 'A',
21701            rankings: 1,
21702            unanimous_initial: true,
21703            deliberated: false,
21704            changed_votes: 0,
21705            unanimous_final: true,
21706            tie_break: None,
21707            judges: 0,
21708            present: 0,
21709            quorum: 0,
21710            met_quorum: true,
21711            uncontested: Some("only candidate A produced a change".to_owned()),
21712        });
21713        state.reviews = vec![ReviewRound {
21714            round: 1,
21715            head: "deadbeef".to_owned(),
21716            verified_head: None,
21717            verified_at: None,
21718            reviews: Vec::new(),
21719            e2e: Vec::new(),
21720            fix: None,
21721            blocking: 0,
21722            answered: 0,
21723            expected: 0,
21724            clean: true,
21725            verify_retried: false,
21726            e2e_deferred: false,
21727            e2e_defer_reason: None,
21728            progressed: false,
21729            vote_split: false,
21730            reconsideration: Vec::new(),
21731            verdict: None,
21732        }];
21733        state.gate = vec![CommandOutcome {
21734            command: "test".to_owned(),
21735            code: Some(0),
21736            output_tail: String::new(),
21737            duration_ms: 0,
21738            resource_blocked: false,
21739        }];
21740        state.gate_ran = true;
21741        // A first pass through `merge` already pushed and opened this pull
21742        // request; `status` is `Landing` because a previous call into `land`
21743        // parked or was interrupted before it reached a terminal outcome.
21744        state.status = RunStatus::Landing;
21745        state.merge = Some(MergeOutcome {
21746            mode: MergeMode::Pr,
21747            ok: true,
21748            detail: "https://example.invalid/x/y/pull/1".to_owned(),
21749            empty: false,
21750        });
21751
21752        // The Landing-resume shortcut calls `run_land` directly rather than
21753        // through `merge`, which is exactly the call site that used to skip
21754        // `settle_questions` - see the fixture below.
21755        ask_test_home();
21756        // A real resume loads a persisted run; driver ownership is checked
21757        // against that record before the Landing shortcut is entered.
21758        state.save().expect("save landing run");
21759        let store = ask::Questions::open();
21760        let q = ask_open_question(&store, &state.id);
21761
21762        let mut runner = Runner {
21763            state,
21764            roles: ResolvedRoles {
21765                implementers: Vec::new(),
21766                judges: Vec::new(),
21767                reviewers: Vec::new(),
21768                fixer: None,
21769                conductor: conductor(),
21770                implementer_roster: Vec::new(),
21771                judge_roster: Vec::new(),
21772                reviewer_roster: Vec::new(),
21773            },
21774            sem: Arc::new(Semaphore::new(1)),
21775            pause: Pause::new(),
21776            interrupt: Pause::new(),
21777        };
21778
21779        // `execute`, not `merge` directly: the Landing-resume shortcut lives
21780        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
21781        // exactly because `review_loop` would otherwise clobber the marker
21782        // first.
21783        runner.execute().await.expect("execute");
21784
21785        assert_eq!(
21786            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21787            Some("https://example.invalid/x/y/pull/1"),
21788            "reentry must not push again or open a second pull request over the \
21789             one `land` is already watching"
21790        );
21791        assert_ne!(
21792            runner.state.status,
21793            RunStatus::Landing,
21794            "land could not actually reach the fake pull request, so it must \
21795             have given up rather than left the run silently parked forever"
21796        );
21797        // `land` could not reach the fake pull request, so it gave up into
21798        // `Blocked` - still resumable, so the question must not have been
21799        // swept just because this branch now also calls `settle_questions`.
21800        assert_eq!(runner.state.status, RunStatus::Blocked);
21801        assert!(
21802            store.get(&q.id).unwrap().status.open(),
21803            "Blocked is still alive; settle_questions must have been a no-op here"
21804        );
21805    }
21806
21807    fn state_with_round(round: ReviewRound) -> RunState {
21808        let mut s = RunState::new(
21809            PathBuf::from("/repo"),
21810            "main".to_owned(),
21811            "abc1234".to_owned(),
21812            "add retries".to_owned(),
21813            Config::default(),
21814        );
21815        s.reviews = vec![round];
21816        s
21817    }
21818
21819    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21820        crate::verdict::Finding {
21821            id: id.to_owned(),
21822            severity,
21823            file: None,
21824            line: None,
21825            title: title.to_owned(),
21826            detail: String::new(),
21827        }
21828    }
21829
21830    #[test]
21831    fn pr_body_names_open_findings_and_declined_ones() {
21832        let round = ReviewRound {
21833            round: 2,
21834            head: "deadbee".to_owned(),
21835            verified_head: None,
21836            verified_at: None,
21837            reviews: vec![ReviewRecord {
21838                attempts: 0,
21839                reviewer: 1,
21840                agent: "alpha".to_owned(),
21841                summary: String::new(),
21842                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21843                vote: None,
21844                failed: None,
21845                duration_ms: 0,
21846            }],
21847            e2e: vec![CommandOutcome {
21848                command: "cargo test".to_owned(),
21849                code: Some(0),
21850                output_tail: String::new(),
21851                duration_ms: 0,
21852                resource_blocked: false,
21853            }],
21854            verify_retried: false,
21855            e2e_deferred: false,
21856            e2e_defer_reason: None,
21857            fix: Some(FixRecord {
21858                agent: "alpha".to_owned(),
21859                addressed: Vec::new(),
21860                rejected: vec![crate::verdict::Rejection {
21861                    id: "R1-1-1".to_owned(),
21862                    why: "not reachable from any caller".to_owned(),
21863                }],
21864                notes: String::new(),
21865                committed: true,
21866                failed: None,
21867                duration_ms: 0,
21868                continuation: None,
21869            }),
21870            blocking: 0,
21871            answered: 1,
21872            expected: 1,
21873            clean: false,
21874            progressed: true,
21875            vote_split: false,
21876            reconsideration: Vec::new(),
21877            verdict: None,
21878        };
21879        let state = state_with_round(round);
21880        let body = pr_message(&state, 'A').body;
21881
21882        assert!(body.contains("add retries"), "the task must still be there");
21883        assert!(body.contains("R2-1-1"), "{body}");
21884        assert!(!body.contains("unused import"), "{body}");
21885        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21886        assert!(
21887            !body.contains("not reachable from any caller"),
21888            "the reason it was declined: {body}"
21889        );
21890    }
21891
21892    #[test]
21893    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21894        let round = ReviewRound {
21895            round: 1,
21896            head: "deadbee".to_owned(),
21897            verified_head: None,
21898            verified_at: None,
21899            reviews: vec![ReviewRecord {
21900                attempts: 0,
21901                reviewer: 1,
21902                agent: "alpha".to_owned(),
21903                summary: String::new(),
21904                findings: Vec::new(),
21905                vote: None,
21906                failed: None,
21907                duration_ms: 0,
21908            }],
21909            e2e: Vec::new(),
21910            verify_retried: false,
21911            e2e_deferred: false,
21912            e2e_defer_reason: None,
21913            fix: None,
21914            blocking: 0,
21915            answered: 1,
21916            expected: 1,
21917            clean: true,
21918            progressed: false,
21919            vote_split: false,
21920            reconsideration: Vec::new(),
21921            verdict: None,
21922        };
21923        let state = state_with_round(round);
21924        let body = pr_message(&state, 'A').body;
21925        assert!(!body.contains("Open review findings"), "{body}");
21926        assert!(!body.contains("Declined"), "{body}");
21927    }
21928
21929    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21930        let mut state = RunState::new(
21931            PathBuf::from("/repo"),
21932            "main".to_owned(),
21933            "abc1234".to_owned(),
21934            instruction.to_owned(),
21935            Config::default(),
21936        );
21937        // These fixtures test execution flow, independent of host disk capacity.
21938        state.config.disk.min_free_bytes = 0;
21939        state.candidates.push(Candidate {
21940            index: 0,
21941            label: 'A',
21942            agent: "alpha".to_owned(),
21943            branch: "magi/x/A".to_owned(),
21944            worktree: PathBuf::from("/wt"),
21945            summary: summary.to_owned(),
21946            stat: String::new(),
21947            files: 1,
21948            commits: 1,
21949            empty: false,
21950            failed: None,
21951            verified_noop: None,
21952            folded: false,
21953            duration_ms: 0,
21954        });
21955        state
21956    }
21957
21958    fn review_state(subjects: &[&str]) -> RunState {
21959        let mut state = state_with_summary(
21960            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21961            "",
21962        );
21963        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21964        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21965        state
21966    }
21967
21968    /// A branch rebased onto a main that moved past the recorded
21969    /// `base_commit` is titled from its own first commit, never main's.
21970    #[tokio::test]
21971    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21972        ask_test_home();
21973        let tmp = tempfile::tempdir().unwrap();
21974        let repo = tmp.path().join("repo");
21975        std::fs::create_dir_all(&repo).unwrap();
21976        init_repo(&repo);
21977        let origin = tmp.path().join("origin.git");
21978        let g = |dir: &Path, args: &[&str]| {
21979            let out = std::process::Command::new("git")
21980                .args(args)
21981                .current_dir(dir)
21982                .quiet()
21983                .output()
21984                .expect("spawn git");
21985            assert!(
21986                out.status.success(),
21987                "git {args:?}: {}",
21988                String::from_utf8_lossy(&out.stderr)
21989            );
21990        };
21991        g(
21992            tmp.path(),
21993            &[
21994                "clone",
21995                "--bare",
21996                "-q",
21997                repo.to_str().unwrap(),
21998                origin.to_str().unwrap(),
21999            ],
22000        );
22001        g(
22002            &repo,
22003            &["remote", "add", "origin", origin.to_str().unwrap()],
22004        );
22005        let c1 = git::rev_parse(&repo, "main").await.unwrap();
22006
22007        // Main moves on; the branch is built on top of the new main.
22008        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
22009        g(&repo, &["add", "-A"]);
22010        g(
22011            &repo,
22012            &["commit", "-q", "-m", "chore(deps): update a crate"],
22013        );
22014        g(&repo, &["push", "-q", "origin", "main"]);
22015        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
22016        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22017        g(&repo, &["add", "-A"]);
22018        g(
22019            &repo,
22020            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
22021        );
22022        g(&repo, &["checkout", "-q", "main"]);
22023
22024        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
22025        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
22026        // Without a readable tracking ref the recorded base's merge base is used.
22027        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
22028        assert_eq!(fallback, c1);
22029
22030        let mut state = review_state(&[
22031            "chore(deps): update a crate",
22032            "fix(daemon): apply a chosen action",
22033        ]);
22034        state.repo = repo.clone();
22035        state.base_branch = "main".to_owned();
22036        state.base_commit = c1;
22037        refresh_reviewed_commits(&mut state, "feat/own").await;
22038        assert_eq!(
22039            state.reviewed_commits,
22040            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
22041        );
22042        assert_eq!(
22043            review_title(&state).as_deref(),
22044            Some("fix(daemon): apply a chosen action")
22045        );
22046        assert_eq!(
22047            leaked_subjects(&state, "feat/own").await,
22048            Some(vec!["chore(deps): update a crate".to_owned()])
22049        );
22050        // A stale tracking ref is still used when the fetch fails, but the
22051        // leak list is withheld.
22052        state.config.merge.remote = "nowhere".to_owned();
22053        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
22054    }
22055
22056    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
22057    /// with `origin/main` one commit behind it, and a summarizing agent that
22058    /// prints whatever `reply` holds.
22059    struct SummaryFixture {
22060        _tmp: tempfile::TempDir,
22061        repo: PathBuf,
22062        reply: PathBuf,
22063        runner: Runner,
22064    }
22065
22066    fn summary_fixture() -> SummaryFixture {
22067        ask_test_home();
22068        let tmp = tempfile::tempdir().unwrap();
22069        let repo = tmp.path().join("repo");
22070        std::fs::create_dir_all(&repo).unwrap();
22071        init_repo(&repo);
22072        let origin = tmp.path().join("origin.git");
22073        let g = |args: &[&str]| {
22074            let out = std::process::Command::new("git")
22075                .args(args)
22076                .current_dir(&repo)
22077                .quiet()
22078                .output()
22079                .expect("spawn git");
22080            assert!(
22081                out.status.success(),
22082                "git {args:?}: {}",
22083                String::from_utf8_lossy(&out.stderr)
22084            );
22085        };
22086        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
22087        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
22088        g(&["push", "-q", "origin", "main"]);
22089        g(&["checkout", "-q", "-b", "feat/own"]);
22090        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22091        g(&["add", "-A"]);
22092        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
22093        let reply = tmp.path().join("reply.txt");
22094        std::fs::write(
22095            &reply,
22096            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
22097        )
22098        .unwrap();
22099        let mut runner = runner_at(RunStatus::Gating);
22100        let mut state = review_state(&["feat(own): add the own file"]);
22101        state.repo = repo.clone();
22102        state.base_branch = "main".to_owned();
22103        state.candidates[0].branch = "feat/own".to_owned();
22104        state.candidates[0].worktree = repo.clone();
22105        let mut author = spec("alpha");
22106        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
22107        author
22108            .env
22109            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
22110        state.config.agents = vec![author];
22111        runner.state = state;
22112        SummaryFixture {
22113            _tmp: tmp,
22114            repo,
22115            reply,
22116            runner,
22117        }
22118    }
22119
22120    async fn posted(f: &mut SummaryFixture) -> PrMessage {
22121        let winner = f.runner.state.candidates[0].clone();
22122        f.runner
22123            .guarded_pr_message(&winner, None, true)
22124            .await
22125            .unwrap()
22126            .expect("an English description never parks")
22127    }
22128
22129    fn clean_round_with_minor_findings() -> ReviewRound {
22130        let mut round = passing_round(2, None);
22131        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
22132        round.reviews = vec![ReviewRecord {
22133            attempts: 0,
22134            reviewer: 1,
22135            agent: "alpha".into(),
22136            summary: "内部レビュー".into(),
22137            findings: vec![
22138                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
22139                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
22140            ],
22141            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
22142            failed: None,
22143            duration_ms: 0,
22144        }];
22145        round
22146    }
22147
22148    #[tokio::test]
22149    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
22150        for review_only in [false, true] {
22151            let mut f = summary_fixture();
22152            if !review_only {
22153                f.runner.state.candidates[0].agent = "alpha".into();
22154            }
22155            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
22156            assert!(f.runner.state.reviews[0].clean);
22157            assert!(f.runner.state.reviews[0].fix.is_none());
22158            assert!(f.runner.state.open_findings().is_empty());
22159            assert_eq!(
22160                crate::pr_outcome::required_finding_ids(&f.runner.state),
22161                ["R2-1-1", "R2-1-2"]
22162            );
22163            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22164            let evidence = crate::pr_outcome::summary_evidence(
22165                &f.runner.state,
22166                crate::pr_outcome::FinalHead::At(&head),
22167            );
22168            assert!(
22169                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
22170            );
22171            // The ordinary reply omits both IDs: no cached description is adopted.
22172            let omitted = posted(&mut f).await;
22173            assert!(omitted.body.contains("No description"));
22174            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
22175            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
22176            assert!(f.runner.state.change_summary.is_none());
22177            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
22178            let described = posted(&mut f).await;
22179            assert!(described.body.contains("improve documentation for callers"));
22180            assert!(described.body.contains("remaining edge case"));
22181            assert!(described.body.contains("## Open review findings"));
22182            assert!(!described.body.contains("未修正"));
22183        }
22184    }
22185
22186    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
22187        let mut f = summary_fixture();
22188        if !review_only {
22189            f.runner.state.candidates[0].agent = "alpha".into();
22190        }
22191        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22192        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
22193        fixed
22194            .fix
22195            .as_mut()
22196            .unwrap()
22197            .rejected
22198            .push(crate::verdict::Rejection {
22199                id: "R1-1-1".into(),
22200                why: "呼び出されないコードのため辞退".into(),
22201            });
22202        let mut clean = clean_round_with_minor_findings();
22203        clean.verified_head = Some(head.clone());
22204        f.runner.state.reviews = vec![fixed, clean];
22205        ok_gate(&mut f.runner.state, Some(&head));
22206        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
22207        f.runner.state.config.agents.push(scripted("beta", reply));
22208        f.runner.state.config.roles.language_judge =
22209            Some(crate::config::AgentChoice::One("beta".into()));
22210        f
22211    }
22212
22213    #[tokio::test]
22214    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
22215        for review_only in [false, true] {
22216            let mut f = rewrite_fixture_with_findings(review_only,
22217                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
22218            let m = posted(&mut f).await;
22219            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22220            let expected =
22221                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
22222            assert!(m.body.contains(&expected));
22223            assert!(m.body.contains("## Review rounds"));
22224            assert!(
22225                m.body
22226                    .contains("Gate (`verify.gate`): passed on the final commit")
22227            );
22228            assert!(
22229                m.body
22230                    .contains("Tests (`verify.e2e`): passed on the final commit")
22231            );
22232            let c = f.runner.state.change_summary.as_ref().unwrap();
22233            assert!(
22234                m.body
22235                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
22236            );
22237            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22238            assert!(crate::pr_outcome::covers_required_findings(
22239                &f.runner.state,
22240                &m.body
22241            ));
22242            assert!(m.body.contains("documentation remains incomplete"));
22243            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
22244            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
22245            assert!(f.runner.state.github_text.is_none());
22246        }
22247    }
22248
22249    #[tokio::test]
22250    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
22251        for review_only in [false, true] {
22252            for reply in [
22253                "TITLE: fix: public description\nA short English description.",
22254                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
22255            ] {
22256                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
22257                let winner = f.runner.state.candidates[0].clone();
22258                let posted = f
22259                    .runner
22260                    .guarded_pr_message(&winner, None, true)
22261                    .await
22262                    .unwrap();
22263                assert!(posted.is_none());
22264                assert!(f.runner.state.parked);
22265                let withheld = f.runner.state.github_text.as_ref().unwrap();
22266                assert!(
22267                    withheld.rewrite.is_none(),
22268                    "an incomplete rewrite must not be selectable"
22269                );
22270                let q = ask::Questions::open()
22271                    .get(withheld.question.as_ref().unwrap())
22272                    .unwrap();
22273                assert!(
22274                    !q.choices
22275                        .contains(&crate::github_text::USE_REWRITE.to_owned())
22276                );
22277                assert!(
22278                    q.detail.contains("R1-1-1")
22279                        && q.detail.contains("R2-1-1")
22280                        && q.detail.contains("R2-1-2")
22281                );
22282            }
22283        }
22284    }
22285
22286    #[tokio::test]
22287    async fn japanese_reports_stay_local_in_both_public_run_modes() {
22288        for review_only in [false, true] {
22289            let mut f = summary_fixture();
22290            if !review_only {
22291                f.runner.state.candidates[0].agent = "alpha".into();
22292            }
22293            f.runner.state.instruction = "内部タスクの説明".into();
22294            f.runner.state.candidates[0].summary = "内部の候補報告".into();
22295            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22296            f.runner.state.reviews = vec![
22297                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
22298                round_with_notes(2, Some("二回目の内部修正報告です。")),
22299                passing_round(3, Some(&head)),
22300            ];
22301            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
22302            ok_gate(&mut f.runner.state, Some(&head));
22303            let captured = f.repo.join("public-prompt.txt");
22304            let author = &mut f.runner.state.config.agents[0];
22305            author.command = vec![
22306                "sh".into(),
22307                "-c".into(),
22308                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22309            ];
22310            author.env.insert(
22311                "PROMPT_FILE".into(),
22312                captured.to_string_lossy().into_owned(),
22313            );
22314            let m = posted(&mut f).await;
22315            assert!(m.body.contains("Adds `own.txt`"));
22316            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
22317            assert!(m.body.contains("R1-1-1"));
22318            assert!(
22319                m.body
22320                    .contains("Gate (`verify.gate`): passed on the final commit")
22321            );
22322            assert!(
22323                m.body
22324                    .contains("Tests (`verify.e2e`): passed on the final commit")
22325            );
22326            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22327            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
22328            let prompt = std::fs::read_to_string(captured).unwrap();
22329            assert!(prompt.contains(&format!(
22330                "git diff {} {head}",
22331                f.runner.state.change_summary.as_ref().unwrap().base
22332            )));
22333            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
22334            assert_eq!(
22335                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
22336                "二回目の内部修正報告です。"
22337            );
22338        }
22339    }
22340
22341    #[tokio::test]
22342    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
22343        for review_only in [false, true] {
22344            let mut f = summary_fixture();
22345            if !review_only {
22346                f.runner.state.candidates[0].agent = "alpha".into();
22347            }
22348            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
22349            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
22350            std::fs::write(&f.reply, "no usable description").unwrap();
22351            let failed = posted(&mut f).await;
22352            assert!(
22353                failed
22354                    .body
22355                    .contains("No description of what this change does could be generated")
22356            );
22357            assert!(failed.body.is_ascii());
22358            assert!(f.runner.state.github_text.is_none());
22359            assert!(f.runner.state.change_summary.is_none());
22360            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22361            let evidence = crate::pr_outcome::summary_evidence(
22362                &f.runner.state,
22363                crate::pr_outcome::FinalHead::At(&head),
22364            );
22365            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
22366            assert!(
22367                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
22368                    .exists()
22369            );
22370            f.runner.state =
22371                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
22372            std::fs::write(
22373                &f.reply,
22374                "TITLE: feat(own): add the own file\nAdds the committed own file.",
22375            )
22376            .unwrap();
22377            let retried = posted(&mut f).await;
22378            assert!(retried.body.contains("Adds the committed own file."));
22379            assert_eq!(
22380                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
22381                "日本語の修正報告"
22382            );
22383        }
22384    }
22385
22386    #[tokio::test]
22387    async fn same_head_verification_and_findings_invalidate_public_prose() {
22388        let mut f = summary_fixture();
22389        posted(&mut f).await;
22390        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22391        let original = f
22392            .runner
22393            .state
22394            .change_summary
22395            .as_ref()
22396            .unwrap()
22397            .evidence
22398            .clone();
22399        ok_gate(&mut f.runner.state, Some(&head));
22400        std::fs::write(
22401            &f.reply,
22402            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
22403        )
22404        .unwrap();
22405        let verified = posted(&mut f).await;
22406        assert!(verified.body.contains("now verified"));
22407        assert_ne!(
22408            f.runner.state.change_summary.as_ref().unwrap().evidence,
22409            original
22410        );
22411        let mut round = round_with_notes(1, Some("内部報告"));
22412        round.reviews.push(ReviewRecord {
22413            attempts: 0,
22414            reviewer: 1,
22415            agent: "alpha".into(),
22416            summary: "内部レビュー".into(),
22417            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
22418            vote: None,
22419            failed: None,
22420            duration_ms: 0,
22421        });
22422        round
22423            .fix
22424            .as_mut()
22425            .unwrap()
22426            .rejected
22427            .push(crate::verdict::Rejection {
22428                id: "R1-1-1".into(),
22429                why: "呼び出し元から到達できない".into(),
22430            });
22431        f.runner.state.reviews = vec![round];
22432        // A description omitting the required ID is unusable, even with a title.
22433        let omitted = posted(&mut f).await;
22434        assert!(omitted.body.contains("No description"));
22435        assert!(omitted.body.contains("R1-1-1"));
22436        assert!(!omitted.body.contains("到達"));
22437        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
22438        let described = posted(&mut f).await;
22439        assert!(described.body.contains("unreachable from callers"));
22440        assert!(described.body.contains("R1-1-1"));
22441        assert!(!described.body.contains("到達"));
22442        let evidence = f
22443            .runner
22444            .state
22445            .change_summary
22446            .as_ref()
22447            .unwrap()
22448            .evidence
22449            .clone();
22450        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22451        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22452        assert!(posted(&mut f).await.body.contains("rationale updated"));
22453        assert_ne!(
22454            f.runner.state.change_summary.as_ref().unwrap().evidence,
22455            evidence
22456        );
22457        // Old serialized summaries have no evidence fingerprint and must regenerate.
22458        f.runner
22459            .state
22460            .change_summary
22461            .as_mut()
22462            .unwrap()
22463            .evidence
22464            .clear();
22465        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22466        assert!(posted(&mut f).await.body.contains("current specification"));
22467    }
22468
22469    #[tokio::test]
22470    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22471        let mut f = summary_fixture();
22472        let m = posted(&mut f).await;
22473        assert_eq!(m.title, "feat(own): add the own file");
22474        assert!(
22475            m.body
22476                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22477            "{}",
22478            m.body
22479        );
22480        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22481        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22482        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22483        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22484        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22485        let kept = f.runner.state.change_summary.as_ref().unwrap();
22486        assert_eq!(
22487            (kept.head.as_str(), kept.base.as_str()),
22488            (head.as_str(), base.as_str())
22489        );
22490        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22491    }
22492
22493    #[tokio::test]
22494    async fn review_only_summary_receives_the_configured_github_language() {
22495        let mut f = summary_fixture();
22496        f.runner.state.config.graph.github_language = "ja".into();
22497        let captured = f.repo.join("summary-prompt.txt");
22498        let author = &mut f.runner.state.config.agents[0];
22499        author.command = vec![
22500            "sh".into(),
22501            "-c".into(),
22502            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22503        ];
22504        author.env.insert(
22505            "PROMPT_FILE".into(),
22506            captured.to_string_lossy().into_owned(),
22507        );
22508        std::fs::write(
22509            &f.reply,
22510            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22511        )
22512        .unwrap();
22513        let m = posted(&mut f).await;
22514        let prompt = std::fs::read_to_string(captured).unwrap();
22515        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22516        assert!(!prompt.contains("in English"));
22517        assert_eq!(m.title, "feat(own): ファイルを追加");
22518        assert!(m.body.contains("own.txt を追加します。"));
22519    }
22520
22521    #[tokio::test]
22522    async fn review_fixes_come_after_the_whole_change_summary() {
22523        let mut f = summary_fixture();
22524        f.runner.state.reviews = vec![
22525            round_with_notes(1, Some("handled the PATH case")),
22526            round_with_notes(2, None),
22527        ];
22528        let m = posted(&mut f).await;
22529        let summary = m.body.find("## Summary").expect("summary");
22530        let fixes = m.body.find("## Review rounds").expect("review rounds");
22531        assert!(summary < fixes, "{}", m.body);
22532        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22533        assert!(m.body.contains("## Final verification"), "{}", m.body);
22534    }
22535
22536    #[tokio::test]
22537    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22538        let mut f = summary_fixture();
22539        let first = posted(&mut f).await;
22540        let first_head = f.runner.state.change_summary.clone().unwrap().head;
22541        // The same head reuses the record: a different reply is not read.
22542        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22543        let again = posted(&mut f).await;
22544        assert_eq!((again.title, again.body), (first.title, first.body));
22545
22546        let g = |args: &[&str]| {
22547            let out = std::process::Command::new("git")
22548                .args(args)
22549                .current_dir(&f.repo)
22550                .quiet()
22551                .output()
22552                .expect("spawn git");
22553            assert!(out.status.success(), "git {args:?}");
22554        };
22555        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22556        g(&["add", "-A"]);
22557        g(&["commit", "-q", "-m", "fix(own): handle more"]);
22558        let second = posted(&mut f).await;
22559        let kept = f.runner.state.change_summary.clone().unwrap();
22560        assert_ne!(kept.head, first_head);
22561        assert!(second.body.contains("Not read yet."), "{}", second.body);
22562        assert!(!second.body.contains("so the first summary is read"));
22563        assert_eq!(second.title, "feat(own): other");
22564    }
22565
22566    #[tokio::test]
22567    async fn an_unavailable_summary_is_stated_and_never_invented() {
22568        let mut f = summary_fixture();
22569        std::fs::write(&f.reply, "no title line here").unwrap();
22570        let m = posted(&mut f).await;
22571        assert!(
22572            m.body
22573                .contains("No description of what this change does could be generated"),
22574            "{}",
22575            m.body
22576        );
22577        assert!(m.body.contains("internal reports remain in local records"));
22578        assert!(f.runner.state.change_summary.is_none());
22579        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22580        // A preview asks nobody and claims nothing.
22581        let winner = f.runner.state.candidates[0].clone();
22582        let preview = f
22583            .runner
22584            .guarded_pr_message(&winner, None, false)
22585            .await
22586            .unwrap()
22587            .unwrap();
22588        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22589    }
22590
22591    #[test]
22592    fn pr_message_review_single_commit_uses_its_subject() {
22593        let state = review_state(&["feat(nats): per-role user"]);
22594        let m = pr_message(&state, 'A');
22595        assert_eq!(m.title, "feat(nats): per-role user");
22596        assert!(!m.body.contains("Review the work already"), "{}", m.body);
22597        assert!(m.body.contains("## Commits under review"), "{}", m.body);
22598    }
22599
22600    #[test]
22601    fn pr_message_review_multi_commit_takes_the_oldest() {
22602        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22603        let m = pr_message(&state, 'A');
22604        assert_eq!(m.title, "feat: the change");
22605        for s in ["feat: the change", "fix: typo", "fix: again"] {
22606            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22607        }
22608    }
22609
22610    #[test]
22611    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22612        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22613            let mut state = review_state(&[first, "fix: later fixup"]);
22614            state.candidates[0].branch = "機能/ログイン".to_owned();
22615            let m = pr_message(&state, 'A');
22616            assert!(
22617                m.title.starts_with("chore: land candidate A of run"),
22618                "{}",
22619                m.title
22620            );
22621        }
22622    }
22623
22624    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22625        BranchFacts {
22626            commits: commits
22627                .iter()
22628                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22629                .collect(),
22630            stat: stat.to_owned(),
22631        }
22632    }
22633
22634    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22635        let mut r = review_round(true, 0, 1, 1, true, true);
22636        r.round = round;
22637        r.fix = notes.map(|n| FixRecord {
22638            agent: "fixer".to_owned(),
22639            addressed: Vec::new(),
22640            rejected: Vec::new(),
22641            notes: n.to_owned(),
22642            committed: true,
22643            failed: None,
22644            duration_ms: 0,
22645            continuation: None,
22646        });
22647        r
22648    }
22649
22650    #[test]
22651    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22652        let state = review_state(&["ignored"]);
22653        let f = facts(
22654            &[
22655                (
22656                    "fix(login): resolve PATH on macOS",
22657                    "Login shells skip rc files.",
22658                ),
22659                ("fix: address review", ""),
22660            ],
22661            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22662        );
22663        let m = pr_message_with(&state, 'A', Some(&f));
22664        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22665        assert!(
22666            m.body
22667                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
22668            "{}",
22669            m.body
22670        );
22671        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22672        assert!(m.body.contains("## Diff stat"), "{}", m.body);
22673        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22674        for banned in [
22675            "Review the work already",
22676            "no task statement",
22677            "Original task",
22678        ] {
22679            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22680        }
22681        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22682    }
22683
22684    #[test]
22685    fn pr_message_review_truncates_a_huge_first_commit_body() {
22686        let state = review_state(&["ignored"]);
22687        let f = facts(
22688            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22689            "s",
22690        );
22691        let m = pr_message_with(&state, 'A', Some(&f));
22692        assert!(m.body.len() < 30_000, "{}", m.body.len());
22693        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22694        assert!(
22695            m.body.contains("- ... 1 more commit(s)"),
22696            "{}",
22697            m.body.len()
22698        );
22699        assert!(m.body.ends_with("magi:candidate-a\n"));
22700    }
22701
22702    #[test]
22703    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22704        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22705        assert_eq!(m.title, "feat: x");
22706        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22707        assert!(!m.body.contains("Diff stat"), "{}", m.body);
22708        assert!(!m.body.contains("no task statement"), "{}", m.body);
22709    }
22710
22711    #[test]
22712    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22713        let mut state = review_state(&["日本語の件名"]);
22714        state.candidates[0].branch = "fix/macos-login-path".to_owned();
22715        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22716        state.candidates[0].branch = "機能/ログイン".to_owned();
22717        assert!(
22718            pr_message(&state, 'A')
22719                .title
22720                .starts_with("chore: land candidate A")
22721        );
22722    }
22723
22724    #[test]
22725    fn pr_message_review_fixes_survive_a_clean_final_round() {
22726        let mut state = review_state(&["feat: x"]);
22727        state.reviews = vec![
22728            round_with_notes(1, Some("handled the PATH case")),
22729            round_with_notes(2, None),
22730        ];
22731        let body = pr_message(&state, 'A').body;
22732        assert!(
22733            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22734            "{body}"
22735        );
22736        assert!(body.contains("### Round 1"), "{body}");
22737
22738        state.reviews = vec![
22739            round_with_notes(1, Some("first fix")),
22740            round_with_notes(2, Some("")),
22741            round_with_notes(3, Some("second fix")),
22742            round_with_notes(4, None),
22743        ];
22744        let body = pr_message(&state, 'A').body;
22745        assert!(
22746            body.contains("### Round 1") && !body.contains("first fix"),
22747            "{body}"
22748        );
22749        assert!(
22750            body.contains("### Round 3") && !body.contains("second fix"),
22751            "{body}"
22752        );
22753        assert!(body.contains("### Round 2"), "{body}");
22754    }
22755
22756    const FINAL: &str = "abcdef1234567890";
22757
22758    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22759        let mut r = round_with_notes(round, None);
22760        r.verified_head = verified.map(str::to_owned);
22761        r
22762    }
22763
22764    fn ok_gate(state: &mut RunState, head: Option<&str>) {
22765        state.gate = vec![CommandOutcome {
22766            command: "gate".to_owned(),
22767            code: Some(0),
22768            output_tail: String::new(),
22769            duration_ms: 0,
22770            resource_blocked: false,
22771        }];
22772        state.gate_ran = true;
22773        state.gate_head = head.map(str::to_owned);
22774    }
22775
22776    fn body_at(state: &RunState) -> String {
22777        pr_message_raw(
22778            state,
22779            'A',
22780            None,
22781            crate::pr_outcome::FinalHead::At(FINAL),
22782            ChangeNote::NotAsked,
22783        )
22784        .body
22785    }
22786
22787    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22788
22789    #[test]
22790    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22791        let mut state = review_state(&["feat: x"]);
22792        state.reviews = vec![
22793            round_with_notes(1, Some(SANDBOXED)),
22794            passing_round(2, Some(FINAL)),
22795        ];
22796        ok_gate(&mut state, Some(FINAL));
22797        let body = body_at(&state);
22798        assert!(!body.contains("Handled the PATH case."), "{body}");
22799        assert!(!body.contains("could not commit"), "{body}");
22800        assert!(!body.contains("could not run"), "{body}");
22801        assert!(
22802            body.contains("The fix is committed on the branch"),
22803            "{body}"
22804        );
22805        assert!(!body.contains("sandbox"), "{body}");
22806        assert!(
22807            body.contains("Tests (`verify.e2e`): passed on the final commit"),
22808            "{body}"
22809        );
22810        assert!(
22811            body.contains("Gate (`verify.gate`): passed on the final commit"),
22812            "{body}"
22813        );
22814        // The raw report is untouched in the record.
22815        assert!(
22816            state.reviews[0]
22817                .fix
22818                .as_ref()
22819                .unwrap()
22820                .notes
22821                .contains("could not commit")
22822        );
22823    }
22824
22825    #[test]
22826    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22827        let mut state = review_state(&["feat: x"]);
22828        let mut r1 = round_with_notes(1, Some(SANDBOXED));
22829        r1.fix.as_mut().unwrap().committed = false;
22830        let mut r2 = passing_round(2, Some(FINAL));
22831        r2.e2e[0].code = Some(1);
22832        state.reviews = vec![r1, r2];
22833        let body = body_at(&state);
22834        assert!(!body.contains("could not commit"), "{body}");
22835        assert!(!body.contains("could not run"), "{body}");
22836        assert!(body.contains("This round produced no commit"), "{body}");
22837        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22838        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22839        assert!(!body.contains("passed"), "{body}");
22840    }
22841
22842    #[test]
22843    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22844        let mut state = review_state(&["feat: x"]);
22845        state.reviews = vec![
22846            round_with_notes(1, Some(SANDBOXED)),
22847            passing_round(2, Some("0123456789abcdef")),
22848        ];
22849        ok_gate(&mut state, Some("0123456789abcdef"));
22850        let body = body_at(&state);
22851        assert!(!body.contains("HTTP tests could not run"), "{body}");
22852        // The commit claim is still settled by the structured record.
22853        assert!(!body.contains("could not commit"), "{body}");
22854        assert!(
22855            body.contains("passed on an earlier commit (`0123456`)"),
22856            "{body}"
22857        );
22858        assert!(!body.contains("passed on the final commit"), "{body}");
22859
22860        // A gate recorded before `gate_head` existed is "not recorded".
22861        ok_gate(&mut state, None);
22862        let body = body_at(&state);
22863        assert!(
22864            body.contains("passed on a commit that was not recorded"),
22865            "{body}"
22866        );
22867    }
22868
22869    #[test]
22870    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22871        let state = state_with_summary(
22872            "add retries to the client",
22873            "TITLE: feat: retries\n\nDid it.",
22874        );
22875        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22876        assert_eq!(m.title, "feat: retries");
22877        assert!(
22878            m.body.contains("<summary>Original task</summary>"),
22879            "{}",
22880            m.body
22881        );
22882        assert!(!m.body.contains("Commits under review"), "{}", m.body);
22883        assert!(
22884            m.body
22885                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22886            "{}",
22887            m.body
22888        );
22889    }
22890
22891    #[test]
22892    fn pr_message_review_bounds_a_long_english_subject() {
22893        let long = format!("feat: {}", "word ".repeat(100));
22894        let m = pr_message(&review_state(&[&long]), 'A');
22895        assert!(m.title.starts_with("feat: word"), "{}", m.title);
22896        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22897    }
22898
22899    #[test]
22900    fn pr_message_implementation_run_is_unchanged_by_review_support() {
22901        let state = state_with_summary("add retries\n\ndetails", "- did some things");
22902        let m = pr_message(&state, 'A');
22903        assert_eq!(m.title, "add retries");
22904        assert!(m.body.contains("<summary>Original task</summary>"));
22905        assert!(!m.body.contains("Commits under review"));
22906        assert_eq!(landing_subject_source(&state), state.instruction);
22907    }
22908
22909    #[test]
22910    fn review_run_squash_subject_is_the_change_not_the_prompt() {
22911        let state = review_state(&["feat: the change", "fix: typo"]);
22912        let source = landing_subject_source(&state);
22913        assert_eq!(land::merge_subject("", &source), "feat: the change");
22914        assert_eq!(
22915            land::merge_subject("magi: candidate A (uncommitted work)", &source),
22916            "feat: the change"
22917        );
22918        // An operator's rename still wins.
22919        assert_eq!(
22920            land::merge_subject("feat: renamed by hand", &source),
22921            "feat: renamed by hand"
22922        );
22923        let blank = review_state(&["日本語"]);
22924        assert!(
22925            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22926        );
22927    }
22928
22929    #[test]
22930    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22931        let state = review_state(&["feat: the change"]);
22932        let source = landing_subject_source(&state);
22933        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22934        assert_eq!(
22935            land::merge_subject(landing_title(&state, old), &source),
22936            "feat: the change"
22937        );
22938        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22939        let task = state_with_summary("add retries", "");
22940        assert_eq!(landing_title(&task, old), old);
22941    }
22942
22943    #[test]
22944    fn pr_message_describes_the_change_not_the_task() {
22945        let state = state_with_summary(
22946            "今回やってほしいこと: results projector を直す",
22947            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22948        );
22949        let m = pr_message(&state, 'A');
22950        assert_eq!(m.title, "fix(web): batch the runs list reads");
22951        assert!(
22952            m.body.starts_with("## Summary\n\n- reads run.json once"),
22953            "{}",
22954            m.body
22955        );
22956        assert!(!m.body.contains("TITLE:"), "{}", m.body);
22957        let task_at = m.body.find("今回やってほしいこと").unwrap();
22958        let details_at = m.body.find("<details>").unwrap();
22959        assert!(
22960            details_at < task_at,
22961            "the task lives inside <details>: {}",
22962            m.body
22963        );
22964        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22965        assert!(m.body.contains("magi:candidate-a"));
22966    }
22967
22968    #[test]
22969    fn pr_message_falls_back_to_the_task_without_a_title_line() {
22970        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22971        let m = pr_message(&state, 'A');
22972        assert_eq!(m.title, "add retries");
22973        assert!(
22974            m.body.contains("## Summary\n\n- did some things"),
22975            "{}",
22976            m.body
22977        );
22978
22979        let none = RunState::new(
22980            PathBuf::from("/repo"),
22981            "main".to_owned(),
22982            "abc1234".to_owned(),
22983            "add retries".to_owned(),
22984            Config::default(),
22985        );
22986        let m = pr_message(&none, 'A');
22987        assert_eq!(m.title, "add retries");
22988        assert!(!m.body.contains("## Summary"), "{}", m.body);
22989    }
22990
22991    #[test]
22992    fn pr_message_refuses_the_candidate_commit_subject() {
22993        for bad in [
22994            "TITLE: magi: candidate A (uncommitted work)",
22995            "TITLE: chore: stuff (uncommitted work)",
22996            "TITLE:   ",
22997        ] {
22998            let state = state_with_summary("add retries", bad);
22999            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
23000        }
23001    }
23002
23003    #[test]
23004    fn pr_message_bounds_a_very_long_task_and_title() {
23005        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
23006        let state = state_with_summary(&long, "- nothing");
23007        let m = pr_message(&state, 'A');
23008        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23009        assert!(!m.title.contains('\n'));
23010
23011        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
23012        let m = pr_message(&state, 'A');
23013        assert!(m.title.starts_with("feat: "));
23014        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23015        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
23016    }
23017
23018    fn long_title_of(instruction: &str) -> String {
23019        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
23020    }
23021
23022    #[test]
23023    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
23024        let first = "Make the landing path keep a readable title for long tasks";
23025        let line = format!(
23026            "{first}. {}",
23027            "And then keep going with more words ".repeat(20)
23028        );
23029        let t = long_title_of(&line);
23030        assert_eq!(t, first);
23031        assert!(!t.starts_with("chore: land"));
23032    }
23033
23034    #[test]
23035    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
23036        let line = "word ".repeat(200);
23037        let t = long_title_of(&line);
23038        assert!(t.ends_with("word..."), "{t}");
23039        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
23040    }
23041
23042    #[test]
23043    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
23044        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
23045            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
23046        }
23047    }
23048
23049    #[test]
23050    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
23051        let mut state = state_with_summary("再試行を追加する", "- nothing");
23052        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
23053        state.config.graph.github_language = "ja".into();
23054        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
23055    }
23056
23057    #[test]
23058    fn pr_message_title_limit_is_exact() {
23059        let at = "a".repeat(PR_TITLE_MAX);
23060        assert_eq!(long_title_of(&at), at);
23061        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
23062        assert!(over.ends_with("..."), "{over}");
23063        assert_eq!(over.chars().count(), PR_TITLE_MAX);
23064    }
23065
23066    #[test]
23067    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
23068        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
23069        let t = long_title_of(&line);
23070        assert!(t.ends_with("..."), "{t}");
23071        assert!(t.is_ascii(), "{t}");
23072    }
23073
23074    #[test]
23075    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
23076        let line = format!(
23077            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
23078            "plus more filler words ".repeat(20)
23079        );
23080        let t = long_title_of(&line);
23081        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
23082    }
23083
23084    #[test]
23085    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
23086        let line = format!("\n\n# {}", "title words ".repeat(40));
23087        let state = state_with_summary(&line, "- nothing");
23088        let m = pr_message(&state, 'A');
23089        assert!(m.title.starts_with("title words"), "{}", m.title);
23090        assert_eq!(
23091            land::merge_subject(&m.title, &landing_subject_source(&state)),
23092            m.title
23093        );
23094        // An operator's rename wins untouched.
23095        assert_eq!(
23096            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
23097            "feat: renamed by hand"
23098        );
23099    }
23100
23101    #[tokio::test]
23102    async fn github_text_rewrite_is_bounded_and_falls_back() {
23103        crate::run::pin_test_home();
23104        for (reply, accepted) in [
23105            (
23106                "TITLE: fix: retries\nAdd retries for failed requests.",
23107                true,
23108            ),
23109            (
23110                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
23111                false,
23112            ),
23113            (
23114                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
23115                true,
23116            ),
23117            (
23118                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
23119                true,
23120            ),
23121            (
23122                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
23123                true,
23124            ),
23125            (
23126                "TITLE: fix: retries\n日本語の説明です。 token=secret",
23127                false,
23128            ),
23129            ("TITLE: fix: retries\n", false),
23130        ] {
23131            let dir = tempfile::tempdir().unwrap();
23132            let mut runner = runner_at(RunStatus::Gating);
23133            runner.state = state_with_summary(
23134                "add retries",
23135                "TITLE: fix: retries\n日本語の説明を書きます。",
23136            );
23137            runner.state.repo = dir.path().to_owned();
23138            runner.state.candidates[0].worktree = dir.path().to_owned();
23139            let mut author = spec("alpha");
23140            author.command = vec![
23141                "sh".into(),
23142                "-c".into(),
23143                "printf '%s' \"$REWRITE_REPLY\"".into(),
23144            ];
23145            author.env.insert("REWRITE_REPLY".into(), reply.into());
23146            runner.state.config.agents = vec![author];
23147            seed_public_summary(&mut runner);
23148            let winner = runner.state.candidates[0].clone();
23149            let first = runner
23150                .guarded_pr_message(&winner, None, true)
23151                .await
23152                .unwrap();
23153            let message = if accepted {
23154                first.expect("a passing rewrite never asks")
23155            } else {
23156                // Withheld: the run parks on a question and posts nothing yet.
23157                assert!(first.is_none());
23158                assert!(runner.state.parked);
23159                let store = ask::Questions::open();
23160                let asked: Vec<_> = store
23161                    .list()
23162                    .into_iter()
23163                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23164                    .collect();
23165                assert_eq!(asked.len(), 1);
23166                assert!(!asked[0].detail.contains("secret"));
23167                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
23168                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
23169                assert_eq!(
23170                    asked[0]
23171                        .choices
23172                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
23173                    rewrite.is_some()
23174                );
23175                // Silence: the question lapses and the neutral text is used,
23176                // without a second rewrite call or a second question.
23177                store
23178                    .update(&asked[0].id, |q| {
23179                        q.abandon("test");
23180                        Ok(())
23181                    })
23182                    .unwrap();
23183                let again = runner
23184                    .guarded_pr_message(&winner, None, true)
23185                    .await
23186                    .unwrap()
23187                    .expect("an abandoned question falls back");
23188                let n = store
23189                    .list()
23190                    .into_iter()
23191                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23192                    .count();
23193                assert_eq!(n, 1, "asked once per rejected text");
23194                again
23195            };
23196            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
23197            assert!(!message.body.contains("secret"));
23198            assert!(!message.body.contains("prefix"));
23199            assert!(!message.title.contains("secret"));
23200            assert_eq!(
23201                message.body.contains("Add retries for failed requests."),
23202                accepted
23203            );
23204            assert_eq!(
23205                runner.state.seats["github-text-rewrite"].turns, 1,
23206                "only one rewrite invocation"
23207            );
23208            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
23209            if accepted {
23210                assert!(runner.state.github_text.is_none());
23211                assert!(!runner.state.parked);
23212                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
23213                assert!(message.body.contains("## Final verification"));
23214                assert!(message.body.contains("Evidence: final diff"));
23215                assert!(
23216                    message
23217                        .body
23218                        .contains(&format!("magi:run/{}", runner.state.id))
23219                );
23220            }
23221            if !accepted {
23222                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23223                assert!(
23224                    message
23225                        .body
23226                        .contains(&format!("magi:run/{}", runner.state.id))
23227                );
23228            }
23229        }
23230    }
23231
23232    fn scripted(id: &str, reply: &str) -> AgentSpec {
23233        let mut a = spec(id);
23234        a.command = vec![
23235            "sh".into(),
23236            "-c".into(),
23237            "printf '%s' \"$REWRITE_REPLY\"".into(),
23238        ];
23239        a.env.insert("REWRITE_REPLY".into(), reply.into());
23240        a
23241    }
23242
23243    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
23244    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
23245
23246    /// Gate tests inject generated public output, not internal reports.
23247    fn seed_public_summary(runner: &mut Runner) {
23248        let repo = &runner.state.repo;
23249        let g = |args: &[&str]| {
23250            let out = std::process::Command::new("git")
23251                .args(args)
23252                .current_dir(repo)
23253                .quiet()
23254                .output()
23255                .unwrap();
23256            assert!(out.status.success(), "git {args:?}");
23257            String::from_utf8_lossy(&out.stdout).trim().to_owned()
23258        };
23259        g(&["init", "-q", "-b", "main"]);
23260        g(&[
23261            "-c",
23262            "user.name=Test",
23263            "-c",
23264            "user.email=test@example.invalid",
23265            "commit",
23266            "-q",
23267            "--allow-empty",
23268            "-m",
23269            "feat: fixture",
23270        ]);
23271        let head = g(&["rev-parse", "HEAD"]);
23272        g(&["update-ref", "refs/remotes/origin/main", &head]);
23273        let evidence = crate::pr_outcome::summary_evidence(
23274            &runner.state,
23275            crate::pr_outcome::FinalHead::At(&head),
23276        );
23277        let generated = &runner.state.candidates[0].summary;
23278        runner.state.change_summary = Some(crate::run::ChangeSummary {
23279            base: head.clone(),
23280            head,
23281            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
23282            title: summary_title(generated).unwrap(),
23283            body: summary_without_title(generated),
23284        });
23285    }
23286
23287    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
23288        let mut runner = runner_at(RunStatus::Gating);
23289        runner.state = state_with_summary("add retries", JA_SUMMARY);
23290        if review_only {
23291            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
23292        }
23293        runner.state.repo = dir.to_owned();
23294        runner.state.candidates[0].worktree = dir.to_owned();
23295        runner.state.config.agents = agents;
23296        seed_public_summary(&mut runner);
23297        runner
23298    }
23299
23300    fn gate_questions(runner: &Runner) -> usize {
23301        ask::Questions::open()
23302            .list()
23303            .into_iter()
23304            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23305            .count()
23306    }
23307
23308    #[tokio::test]
23309    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
23310        crate::run::pin_test_home();
23311        let dir = tempfile::tempdir().unwrap();
23312        let mut runner = rewrite_runner(
23313            dir.path(),
23314            true,
23315            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23316        );
23317        runner.state.config.roles.language_judge =
23318            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
23319        let winner = runner.state.candidates[0].clone();
23320        let msg = runner
23321            .guarded_pr_message(&winner, None, true)
23322            .await
23323            .unwrap()
23324            .expect("an English rewrite posts without asking");
23325        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
23326        assert!(msg.body.contains("<details>kept</details>"));
23327        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
23328        assert_eq!(gate_questions(&runner), 0);
23329        assert!(runner.state.github_text.is_none());
23330        assert!(!runner.state.seats.contains_key("impl-A"));
23331        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23332    }
23333
23334    #[tokio::test]
23335    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
23336        crate::run::pin_test_home();
23337        let dir = tempfile::tempdir().unwrap();
23338        let mut runner = rewrite_runner(
23339            dir.path(),
23340            false,
23341            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23342        );
23343        runner.state.config.roles.language_judge =
23344            Some(crate::config::AgentChoice::One("beta".into()));
23345        runner.state.seats.insert(
23346            "impl-A".into(),
23347            SeatState::new("impl-A", "alpha", runner.state.seed),
23348        );
23349        let winner = runner.state.candidates[0].clone();
23350        let msg = runner
23351            .guarded_pr_message(&winner, None, true)
23352            .await
23353            .unwrap()
23354            .expect("rewritten");
23355        assert!(msg.body.contains("<details>kept</details>"));
23356        assert_eq!(runner.state.seats["impl-A"].turns, 0);
23357        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23358        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23359    }
23360
23361    #[tokio::test]
23362    async fn rewrite_reuses_a_resumable_implementer_session() {
23363        crate::run::pin_test_home();
23364        let dir = tempfile::tempdir().unwrap();
23365        let mut runner = rewrite_runner(
23366            dir.path(),
23367            false,
23368            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
23369        );
23370        runner.state.config.roles.language_judge =
23371            Some(crate::config::AgentChoice::One("beta".into()));
23372        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23373        seat.turns = 1;
23374        runner.state.seats.insert("impl-A".into(), seat);
23375        let winner = runner.state.candidates[0].clone();
23376        runner
23377            .guarded_pr_message(&winner, None, true)
23378            .await
23379            .unwrap()
23380            .expect("rewritten");
23381        assert_eq!(runner.state.seats["impl-A"].turns, 2);
23382        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
23383    }
23384
23385    #[tokio::test]
23386    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
23387        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
23388        // session, was out of quota, and nothing else was asked.
23389        crate::run::pin_test_home();
23390        let dir = tempfile::tempdir().unwrap();
23391        let mut exhausted = spec("alpha");
23392        exhausted.command = vec!["false".into()];
23393        let mut runner = rewrite_runner(
23394            dir.path(),
23395            false,
23396            vec![exhausted, scripted("beta", EN_REWRITE)],
23397        );
23398        // The implementer's own agent is in the role too: it is not asked twice.
23399        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23400            "alpha".into(),
23401            "beta".into(),
23402        ]));
23403        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23404        seat.turns = 1;
23405        runner.state.seats.insert("impl-A".into(), seat);
23406        let winner = runner.state.candidates[0].clone();
23407        let msg = runner
23408            .guarded_pr_message(&winner, None, true)
23409            .await
23410            .unwrap()
23411            .expect("the role rewrites after the implementer fails");
23412        assert!(msg.body.contains("`src/net.rs`"));
23413        assert_eq!(
23414            runner.state.seats["impl-A"].turns, 2,
23415            "the implementer was tried once"
23416        );
23417        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23418        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23419    }
23420
23421    #[tokio::test]
23422    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
23423        crate::run::pin_test_home();
23424        let dir = tempfile::tempdir().unwrap();
23425        let mut broken = spec("bad");
23426        broken.command = vec!["false".into()];
23427        let mut runner = rewrite_runner(
23428            dir.path(),
23429            true,
23430            vec![
23431                broken,
23432                scripted("noformat", "Add retries."),
23433                scripted("good", EN_REWRITE),
23434            ],
23435        );
23436        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23437            "bad".into(),
23438            "noformat".into(),
23439            "good".into(),
23440        ]));
23441        let winner = runner.state.candidates[0].clone();
23442        let msg = runner
23443            .guarded_pr_message(&winner, None, true)
23444            .await
23445            .unwrap()
23446            .expect("the third agent rewrites");
23447        assert!(msg.body.contains("`src/net.rs`"));
23448        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23449        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23450    }
23451
23452    #[tokio::test]
23453    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23454        crate::run::pin_test_home();
23455        // (agents, role): still Japanese; no TITLE; nothing resolvable.
23456        for (agents, role, saved) in [
23457            (
23458                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23459                Some("beta"),
23460                true,
23461            ),
23462            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23463            (Vec::new(), None, false),
23464        ] {
23465            let dir = tempfile::tempdir().unwrap();
23466            let mut runner = rewrite_runner(dir.path(), true, agents);
23467            runner.state.config.roles.language_judge =
23468                role.map(|r| crate::config::AgentChoice::One(r.into()));
23469            let winner = runner.state.candidates[0].clone();
23470            let first = runner
23471                .guarded_pr_message(&winner, None, true)
23472                .await
23473                .unwrap();
23474            assert!(first.is_none(), "withheld text parks the run");
23475            assert!(runner.state.parked);
23476            assert_eq!(gate_questions(&runner), 1);
23477            let g = runner.state.github_text.as_ref().unwrap();
23478            assert_eq!(g.rewrite.is_some(), saved);
23479            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23480        }
23481    }
23482
23483    #[tokio::test]
23484    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23485        crate::run::pin_test_home();
23486        let dir = tempfile::tempdir().unwrap();
23487        let mut runner = runner_at(RunStatus::Gating);
23488        runner.state =
23489            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23490        runner.state.repo = dir.path().to_owned();
23491        runner.state.candidates[0].worktree = dir.path().to_owned();
23492        let mut author = spec("alpha");
23493        author.command = vec![
23494            "sh".into(),
23495            "-c".into(),
23496            "printf '%s' \"$REWRITE_REPLY\"".into(),
23497        ];
23498        author.env.insert(
23499            "REWRITE_REPLY".into(),
23500            format!(
23501                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23502                "English sentence. ".repeat(600)
23503            ),
23504        );
23505        runner.state.config.agents = vec![author];
23506        seed_public_summary(&mut runner);
23507        let winner = runner.state.candidates[0].clone();
23508        assert!(
23509            runner
23510                .guarded_pr_message(&winner, None, true)
23511                .await
23512                .unwrap()
23513                .is_none()
23514        );
23515        let record = runner.state.github_text.clone().unwrap();
23516        let rewrite = record.rewrite.clone().unwrap();
23517        assert_eq!(rewrite.failed, ["body-language"]);
23518        assert!(
23519            rewrite
23520                .body
23521                .ends_with(&format!("magi:run/{}", runner.state.id))
23522        );
23523        let store = ask::Questions::open();
23524        let q = store.get(record.question.as_ref().unwrap()).unwrap();
23525        assert!(
23526            q.choices
23527                .contains(&crate::github_text::USE_REWRITE.to_owned())
23528        );
23529        for label in [
23530            "ORIGINAL",
23531            "FALLBACK",
23532            "REWRITE",
23533            "Rules still failed: body-language",
23534            "truncated",
23535        ] {
23536            assert!(q.detail.contains(label), "{label}");
23537        }
23538        let artifact = std::fs::read_to_string(crate::run::artifact_path(
23539            &runner.state,
23540            &crate::github_text::artifact_name(&record.fingerprint),
23541        ))
23542        .unwrap();
23543        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23544        // Restart from serialized state, then retry after a bad owner title.
23545        runner.state =
23546            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23547        store
23548            .update(&q.id, |q| {
23549                q.say("まだ日本語")?;
23550                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23551            })
23552            .unwrap();
23553        assert!(
23554            runner
23555                .guarded_pr_message(&winner, None, true)
23556                .await
23557                .unwrap()
23558                .is_none()
23559        );
23560        let retry = runner.state.github_text.as_ref().unwrap();
23561        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23562        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23563        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23564        assert!(
23565            q.posting_comparison
23566                .as_ref()
23567                .unwrap()
23568                .matches_detail(&q.detail)
23569        );
23570        store
23571            .update(&q.id, |q| {
23572                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23573            })
23574            .unwrap();
23575        let message = runner
23576            .guarded_pr_message(&winner, None, true)
23577            .await
23578            .unwrap()
23579            .unwrap();
23580        assert_eq!(message.title, rewrite.title);
23581        assert_eq!(message.body, rewrite.body);
23582        assert_eq!(
23583            runner.state.github_text.as_ref().unwrap().choice,
23584            Some(GithubTextChoice::Rewrite)
23585        );
23586        let again = runner
23587            .guarded_pr_message(&winner, None, true)
23588            .await
23589            .unwrap()
23590            .unwrap();
23591        assert_eq!(again.title, message.title);
23592        assert_eq!(again.body, message.body);
23593        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23594        assert!(
23595            !runner
23596                .state
23597                .events
23598                .iter()
23599                .any(|e| e.message.contains(&rewrite.title)
23600                    || e.message.contains("English sentence."))
23601        );
23602    }
23603
23604    #[tokio::test]
23605    async fn historical_posting_reply_never_expires_into_fallback() {
23606        crate::run::pin_test_home();
23607        let dir = tempfile::tempdir().unwrap();
23608        let mut runner = runner_at(RunStatus::Gating);
23609        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23610        runner.state.repo = dir.path().to_owned();
23611        runner.state.candidates[0].worktree = dir.path().to_owned();
23612        seed_public_summary(&mut runner);
23613        let winner = runner.state.candidates[0].clone();
23614        assert!(
23615            runner
23616                .guarded_pr_message(&winner, None, true)
23617                .await
23618                .unwrap()
23619                .is_none()
23620        );
23621        let id = runner
23622            .state
23623            .github_text
23624            .as_ref()
23625            .unwrap()
23626            .question
23627            .clone()
23628            .unwrap();
23629        let store = ask::Questions::open();
23630        let mut q = store.get(&id).unwrap();
23631        q.asked_at = "2020-01-01T00:00:00Z".parse().unwrap();
23632        q.answer(ask::Answer::Choice(
23633            crate::github_text::POST_ORIGINAL.into(),
23634        ))
23635        .unwrap();
23636        q.answer_delivered = true;
23637        store.put(&mut q).unwrap();
23638        let source = std::fs::read(store.path_of(&id)).unwrap();
23639        let migration = crate::run::home().join("notifications/task-chat-migration-v1");
23640        std::fs::create_dir_all(&migration).unwrap();
23641        std::fs::write(
23642            migration.join("state.json"),
23643            serde_json::to_vec(&serde_json::json!({
23644                "schema": 1, "sources": [id], "pending": [id]
23645            }))
23646            .unwrap(),
23647        )
23648        .unwrap();
23649        for _ in 0..2 {
23650            assert!(
23651                runner
23652                    .guarded_pr_message(&winner, None, true)
23653                    .await
23654                    .unwrap()
23655                    .is_none()
23656            );
23657            assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23658            assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23659        }
23660        let home = crate::run::home();
23661        crate::task_chat::sweep(
23662            &crate::queue::Queue::at(home.join("queue")),
23663            &crate::talk::Talks::at(home.join("talks")),
23664            &store,
23665            &home,
23666        );
23667        let outbox = crate::task_inquiry::Outbox::at(&home);
23668        outbox
23669            .update_decision(&id, |d| {
23670                d.accept(&format!("decision: {}", crate::github_text::POST_ORIGINAL))?;
23671                d.source_answer_at = q.answered_at;
23672                Ok(true)
23673            })
23674            .unwrap();
23675        outbox
23676            .settle(
23677                &id,
23678                "owner-chat",
23679                "Owner confirms no publication remains necessary",
23680                None,
23681            )
23682            .unwrap();
23683        assert!(
23684            runner
23685                .guarded_pr_message(&winner, None, true)
23686                .await
23687                .unwrap()
23688                .is_none()
23689        );
23690        assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23691        assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23692    }
23693
23694    #[tokio::test]
23695    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23696        crate::run::pin_test_home();
23697        let dir = tempfile::tempdir().unwrap();
23698        let mut runner = runner_at(RunStatus::Gating);
23699        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23700        runner.state.repo = dir.path().to_owned();
23701        runner.state.candidates[0].worktree = dir.path().to_owned();
23702        seed_public_summary(&mut runner);
23703        let winner = runner.state.candidates[0].clone();
23704        assert!(
23705            runner
23706                .guarded_pr_message(&winner, None, true)
23707                .await
23708                .unwrap()
23709                .is_none()
23710        );
23711        let record = runner.state.github_text.as_ref().unwrap();
23712        assert!(record.rewrite.is_none());
23713        // Records written before the additive field deserialize with no rewrite.
23714        let mut old = serde_json::to_value(record).unwrap();
23715        old.as_object_mut().unwrap().remove("rewrite");
23716        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23717        assert!(old.rewrite.is_none());
23718        let id = record.question.clone().unwrap();
23719        ask::Questions::open()
23720            .update(&id, |q| {
23721                q.choices.push(crate::github_text::USE_REWRITE.into());
23722                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23723            })
23724            .unwrap();
23725        let message = runner
23726            .guarded_pr_message(&winner, None, true)
23727            .await
23728            .unwrap()
23729            .unwrap();
23730        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23731        assert_eq!(
23732            runner.state.github_text.as_ref().unwrap().choice,
23733            Some(GithubTextChoice::Fallback)
23734        );
23735    }
23736
23737    #[tokio::test]
23738    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23739        crate::run::pin_test_home();
23740        let dir = tempfile::tempdir().unwrap();
23741        let mut runner = runner_at(RunStatus::Gating);
23742        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23743        runner.state.repo = dir.path().to_owned();
23744        runner.state.candidates[0].worktree = dir.path().to_owned();
23745        seed_public_summary(&mut runner);
23746        let winner = runner.state.candidates[0].clone();
23747        assert!(
23748            runner
23749                .guarded_pr_message(&winner, None, true)
23750                .await
23751                .unwrap()
23752                .is_none()
23753        );
23754        let store = ask::Questions::open();
23755        let find = |run: &str| {
23756            store
23757                .list()
23758                .into_iter()
23759                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23760                .unwrap()
23761        };
23762        let run = runner.state.id.clone();
23763        let q = find(&run);
23764        let fp = runner
23765            .state
23766            .github_text
23767            .as_ref()
23768            .unwrap()
23769            .fingerprint
23770            .clone();
23771        let name = crate::github_text::artifact_name(&fp);
23772        let path = crate::run::artifact_path(&runner.state, &name);
23773        assert!(q.detail.contains(&path.display().to_string()));
23774        assert!(q.detail.contains("title-language: title"));
23775        let saved = std::fs::read_to_string(&path).expect("artifact written");
23776        assert!(saved.contains("日本語のタイトル"));
23777        assert!(q.cwd.is_none());
23778        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23779        // A bad replacement is refused and asked about once more.
23780        let say = |id: &str, text: &str, choice: &str| {
23781            store
23782                .update(id, |q| {
23783                    q.thread.push(ask::Turn {
23784                        who: ask::Who::Operator,
23785                        body: text.to_owned(),
23786                        at: jiff::Timestamp::now(),
23787                        note: None,
23788                    });
23789                    q.answer(ask::Answer::Choice(choice.to_owned()))
23790                })
23791                .unwrap();
23792        };
23793        say(&q.id, "まだ日本語", "use my text");
23794        assert!(
23795            runner
23796                .guarded_pr_message(&winner, None, true)
23797                .await
23798                .unwrap()
23799                .is_none()
23800        );
23801        let second = find(&run);
23802        assert_ne!(second.id, q.id);
23803        assert!(second.detail.contains("did not pass"));
23804        // The retry still compares the original text; the rejected
23805        // replacement is named by category only.
23806        assert!(second.detail.contains("日本語のタイトル"));
23807        assert!(!second.detail.contains("まだ日本語"));
23808        assert!(second.detail.contains(&path.display().to_string()));
23809        say(&second.id, "fix: retry failed requests", "use my text");
23810        let message = runner
23811            .guarded_pr_message(&winner, None, true)
23812            .await
23813            .unwrap()
23814            .expect("a vetted replacement is posted");
23815        assert_eq!(message.title, "fix: retry failed requests");
23816        assert_eq!(
23817            runner
23818                .state
23819                .github_text
23820                .as_ref()
23821                .unwrap()
23822                .chosen_title
23823                .as_deref(),
23824            Some("fix: retry failed requests")
23825        );
23826        assert!(runner.state.github_text.as_ref().unwrap().resolved);
23827        // Resumed again: the saved decision is applied, nothing is re-asked.
23828        let again = runner
23829            .guarded_pr_message(&winner, None, true)
23830            .await
23831            .unwrap()
23832            .unwrap();
23833        assert_eq!(again.title, message.title);
23834        assert_eq!(
23835            store
23836                .list()
23837                .into_iter()
23838                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23839                .count(),
23840            2
23841        );
23842    }
23843
23844    #[test]
23845    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23846        // What magi itself writes stays English under any configured language,
23847        // so a future localisation of these headings fails here. (The agents'
23848        // own text is also checked by the posting gate.)
23849        let mut state = state_with_summary(
23850            "add retries",
23851            "TITLE: fix(web): batch reads\n- reads run.json once",
23852        );
23853        state.config.graph.language = "ja".to_owned();
23854        let m = pr_message(&state, 'A');
23855        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23856        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23857
23858        // The task is the operator's own text: it goes in untouched, and the
23859        // fallback title (no summary) may be in its language too.
23860        let task = "今回やってほしいこと: results projector を直す";
23861        let mut state = state_with_summary(task, "- no title line");
23862        state.config.graph.language = "ja".to_owned();
23863        let m = pr_message(&state, 'A');
23864        assert_eq!(
23865            m.title,
23866            format!("chore: land candidate A of run {}", state.id)
23867        );
23868        assert!(
23869            m.body.contains(&format!(
23870                "<summary>Original task</summary>\n\n{task}\n\n</details>"
23871            )),
23872            "{}",
23873            m.body
23874        );
23875    }
23876
23877    #[test]
23878    fn pr_message_scrubs_home_paths_and_addresses() {
23879        let state = state_with_summary(
23880            "fix it in /Users/someone/src/x",
23881            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23882        );
23883        let m = pr_message(&state, 'A');
23884        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23885            assert!(!m.body.contains(leak), "{}", m.body);
23886        }
23887        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23888    }
23889
23890    #[test]
23891    fn pr_message_survives_a_task_that_closes_details() {
23892        let state = state_with_summary("a </details> b", "TITLE: fix: x");
23893        let m = pr_message(&state, 'A');
23894        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23895    }
23896
23897    #[test]
23898    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23899        let cmd = manual_merge_command(
23900            MergeStyle::Squash,
23901            Path::new("/repo"),
23902            "b",
23903            "fix: \"quoted\" $(x) `y`\n\nbody",
23904        );
23905        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23906    }
23907
23908    #[test]
23909    fn manual_merge_command_matches_the_configured_style() {
23910        let repo = Path::new("/repo");
23911        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23912
23913        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23914        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23915
23916        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23917        assert_eq!(
23918            squash,
23919            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23920             \"Merge magi run 0832 (candidate A)\""
23921        );
23922
23923        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23924        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23925    }
23926
23927    #[test]
23928    fn a_nudge_gets_a_quarter_of_the_budget() {
23929        // The judge and implement budgets magi ships with.
23930        assert_eq!(retry_budget(secs(1200), true), secs(300));
23931        assert_eq!(retry_budget(secs(3600), true), secs(900));
23932    }
23933
23934    #[test]
23935    fn a_resent_prompt_keeps_the_whole_budget() {
23936        // The seat kept no context, so the retry is the original job again and
23937        // shortening it would only guarantee a second failure.
23938        assert_eq!(retry_budget(secs(1200), false), secs(1200));
23939        assert_eq!(retry_budget(secs(60), false), secs(60));
23940    }
23941
23942    #[test]
23943    fn the_floor_never_exceeds_the_original_budget() {
23944        // A short configured timeout must not be *raised* by the floor: the
23945        // operator asked for a bound, and a retry may not outlast the attempt
23946        // it is retrying.
23947        assert_eq!(retry_budget(secs(60), true), secs(60));
23948        assert_eq!(retry_budget(secs(480), true), secs(120));
23949        assert_eq!(retry_budget(secs(0), true), secs(0));
23950    }
23951
23952    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23953        agent::CommandEvidence {
23954            id: "item1".to_owned(),
23955            description: "cargo test".to_owned(),
23956            exit_code,
23957            result_summary: String::new(),
23958            source: "codex".to_owned(),
23959        }
23960    }
23961
23962    #[test]
23963    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23964        // No evidence is not the same fact as unconfirmed evidence: a
23965        // backend with no adapter, or a reply that ran no commands at all,
23966        // must not be misread as carrying a dangling job.
23967        assert!(!has_unconfirmed_command(&[]));
23968    }
23969
23970    #[test]
23971    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23972        // Deliberately not a check on the exit code's *value*: a fixer
23973        // legitimately runs something that fails mid-iteration before it
23974        // succeeds, and that must never by itself reopen a valid report.
23975        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23976        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23977        assert!(!has_unconfirmed_command(&[
23978            evidence(Some(0)),
23979            evidence(Some(101))
23980        ]));
23981    }
23982
23983    #[test]
23984    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23985        assert!(has_unconfirmed_command(&[
23986            evidence(Some(0)),
23987            evidence(None)
23988        ]));
23989    }
23990
23991    #[test]
23992    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23993        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23994        assert_eq!(
23995            verified_noop_claim(true, &[], text).as_deref(),
23996            Some("already fixed by b32cfc4, on main.")
23997        );
23998    }
23999
24000    #[test]
24001    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
24002        // A timeout or a bad exit code reads as the ordinary loss it is,
24003        // whatever the reply's own prose claims.
24004        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24005        assert!(verified_noop_claim(false, &[], text).is_none());
24006    }
24007
24008    #[test]
24009    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
24010        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24011        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
24012        // A confirmed command alongside the marker is fine.
24013        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
24014    }
24015
24016    #[test]
24017    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
24018        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
24019    }
24020
24021    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
24022    /// pair, in order, labelled A, B, C, ...
24023    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
24024        runner.state.candidates = shape
24025            .iter()
24026            .enumerate()
24027            .map(|(i, &(empty, verified))| Candidate {
24028                index: i,
24029                label: (b'A' + i as u8) as char,
24030                agent: "sonnet".to_owned(),
24031                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
24032                worktree: PathBuf::from(format!("/wt/{i}")),
24033                summary: String::new(),
24034                stat: String::new(),
24035                files: 0,
24036                commits: 0,
24037                empty,
24038                failed: None,
24039                verified_noop: verified.map(str::to_owned),
24040                duration_ms: 0,
24041                folded: false,
24042            })
24043            .collect();
24044    }
24045
24046    #[test]
24047    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
24048        ask_test_home();
24049        let mut runner = runner_at(RunStatus::Implementing);
24050        set_candidates(
24051            &mut runner,
24052            &[
24053                (true, Some("already on main at b32cfc4")),
24054                (true, Some("same fix, see the existing test")),
24055            ],
24056        );
24057
24058        runner
24059            .after_implement()
24060            .expect("a verified no-op is not an error");
24061
24062        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
24063    }
24064
24065    #[test]
24066    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
24067        ask_test_home();
24068        let mut runner = runner_at(RunStatus::Implementing);
24069        // Candidate A declares a verified no-op; candidate B simply wrote
24070        // nothing and said nothing about why. One candidate's claim is not
24071        // the whole run's agreement.
24072        set_candidates(
24073            &mut runner,
24074            &[(true, Some("already on main at b32cfc4")), (true, None)],
24075        );
24076
24077        let err = runner
24078            .after_implement()
24079            .expect_err("an unverified empty candidate must still fail the run");
24080
24081        assert!(
24082            err.to_string().contains("no candidate produced a change"),
24083            "{err}"
24084        );
24085        assert_eq!(runner.state.status, RunStatus::Failed);
24086    }
24087
24088    #[test]
24089    fn after_implement_still_fails_an_ordinary_all_empty_run() {
24090        ask_test_home();
24091        let mut runner = runner_at(RunStatus::Implementing);
24092        set_candidates(&mut runner, &[(true, None), (true, None)]);
24093
24094        let err = runner
24095            .after_implement()
24096            .expect_err("no candidate declared anything; this is an ordinary failure");
24097
24098        assert!(
24099            err.to_string().contains("no candidate produced a change"),
24100            "{err}"
24101        );
24102        assert_eq!(runner.state.status, RunStatus::Failed);
24103    }
24104}
24105
24106/// A graft rollback whose restore could not be confirmed (see
24107/// [`Runner::graft`]).
24108#[derive(Debug)]
24109struct GraftRestoreFailed(String);
24110
24111impl std::fmt::Display for GraftRestoreFailed {
24112    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
24113        write!(
24114            f,
24115            "could not restore the winner to its pre-graft commit: {}",
24116            self.0
24117        )
24118    }
24119}
24120
24121impl std::error::Error for GraftRestoreFailed {}
24122
24123/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
24124/// not the agent: they never draw on an agent's own timeout, and running out of
24125/// them never reads as an agent failure.
24126#[cfg(test)]
24127mod resource_wait_tests {
24128    use super::*;
24129    use std::collections::BTreeMap;
24130
24131    fn sh(id: &str, script: &str) -> AgentSpec {
24132        AgentSpec {
24133            id: id.to_owned(),
24134            kind: crate::config::AgentKind::Command,
24135            model: None,
24136            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
24137            extra_args: Vec::new(),
24138            env: BTreeMap::new(),
24139            prompt_delivery: None,
24140            max_turn_secs: None,
24141        }
24142    }
24143
24144    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
24145        SeatJob {
24146            seat: SeatState::new("impl-A", &spec.id, 1),
24147            spec,
24148            cwd: dir.to_path_buf(),
24149            prompt: "do it".to_owned(),
24150            timeout,
24151            allow_write: true,
24152            sessions: false,
24153            artifacts: dir.join("art"),
24154            stem: "impl-A".to_owned(),
24155            handover: None,
24156        }
24157    }
24158
24159    fn state(resource_wait_secs: u64) -> RunState {
24160        let mut config = Config::default();
24161        config.graph.timeout_resource_wait = resource_wait_secs;
24162        RunState::new(
24163            PathBuf::from("/repo"),
24164            "main".to_owned(),
24165            "abc1234".to_owned(),
24166            "task".to_owned(),
24167            config,
24168        )
24169    }
24170
24171    fn hold(cache: &Path) -> crate::cache::Guard {
24172        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
24173        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
24174            crate::cache::AcquireOutcome::Acquired(g) => g,
24175            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
24176        }
24177    }
24178
24179    async fn run(
24180        jobs: Vec<SeatJob>,
24181        sem: Arc<Semaphore>,
24182        cache: Option<&Path>,
24183        st: &mut RunState,
24184    ) -> Vec<(usize, SeatState, AgentOutcome)> {
24185        let prompts = Prompts::default();
24186        let ctx = WaveCtx {
24187            run: "r",
24188            node: "implement",
24189            prompts: &prompts,
24190            cache,
24191            round: None,
24192            carry_seats: false,
24193            pauses: None,
24194        };
24195        wave(jobs, sem, &ctx, st, 0).await
24196    }
24197
24198    #[tokio::test]
24199    async fn verification_cache_wait_preserves_the_command_budget() {
24200        crate::run::pin_test_home();
24201        let tmp = tempfile::tempdir().unwrap();
24202        let cache = tmp.path().join("target");
24203        let guard = hold(&cache);
24204        let release = tokio::spawn(async move {
24205            tokio::time::sleep(Duration::from_millis(100)).await;
24206            guard.release();
24207        });
24208        let mut st = state(10);
24209        let budget = Duration::from_millis(50);
24210        let wait = repository_wait(&st, None);
24211        let (outcomes, _) = with_cache_lease(
24212            &mut st,
24213            Some(&cache),
24214            "gate",
24215            "gate",
24216            tmp.path(),
24217            "abc1234",
24218            budget,
24219            "test gate",
24220            &wait,
24221            |_, available| async move {
24222                assert_eq!(
24223                    available, budget,
24224                    "contention must not shorten verification"
24225                );
24226                (Vec::new(), false, Vec::new())
24227            },
24228        )
24229        .await
24230        .unwrap();
24231        release.await.unwrap();
24232        assert!(outcomes.is_empty(), "{outcomes:?}");
24233        assert!(
24234            st.events
24235                .iter()
24236                .any(|event| event.message.contains("waiting for"))
24237        );
24238        assert!(st.resource_wait.is_none());
24239        // The verification body no longer owns the lease after returning.
24240        hold(&cache).release();
24241    }
24242
24243    #[tokio::test]
24244    async fn verification_cache_wait_expires_without_starting_commands() {
24245        crate::run::pin_test_home();
24246        let tmp = tempfile::tempdir().unwrap();
24247        let cache = tmp.path().join("target");
24248        let _held = hold(&cache);
24249        let mut st = state(0);
24250        let wait = repository_wait(&st, None);
24251        let (outcomes, _) = with_cache_lease(
24252            &mut st,
24253            Some(&cache),
24254            "gate",
24255            "gate",
24256            tmp.path(),
24257            "abc1234",
24258            Duration::from_secs(60),
24259            "test gate",
24260            &wait,
24261            |_, _| async { panic!("commands must not start without the cache lease") },
24262        )
24263        .await
24264        .unwrap();
24265        assert_eq!(outcomes.len(), 1);
24266        assert!(outcomes[0].resource_blocked);
24267        assert!(st.resource_wait.is_none());
24268        assert!(matches!(
24269            st.resource_block.unwrap().cause,
24270            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
24271        ));
24272    }
24273
24274    #[tokio::test]
24275    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
24276        crate::run::pin_test_home();
24277        let tmp = tempfile::tempdir().unwrap();
24278        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
24279        let cache = tmp.path().join("cache");
24280        let mut state = state(21600);
24281        let wait = repository_wait(&state, None);
24282        let (outcomes, _) = with_cache_lease(
24283            &mut state,
24284            Some(&cache),
24285            "gate",
24286            "gate",
24287            tmp.path(),
24288            "head",
24289            Duration::from_secs(60),
24290            "gate",
24291            &wait,
24292            |_, _| async { panic!("freshness failure must not start verification") },
24293        )
24294        .await
24295        .unwrap();
24296        assert_eq!(outcomes.len(), 1);
24297        assert!(outcomes[0].resource_blocked);
24298        let block = state.resource_block.unwrap();
24299        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
24300        assert!(!block.description().contains("timed out"));
24301    }
24302
24303    #[tokio::test]
24304    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
24305        crate::run::pin_test_home();
24306        let tmp = tempfile::tempdir().unwrap();
24307        let cache = tmp.path().join("cache");
24308        let _held = hold(&cache);
24309        let mut state = state(21600);
24310        let pause = Pause::new();
24311        let interrupt = Pause::new();
24312        let cause = ParkCause::Operator;
24313        pause.park_for(cause.clone());
24314        let prompts = Prompts::default();
24315        let ctx = WaveCtx {
24316            run: "test",
24317            node: "fix",
24318            prompts: &prompts,
24319            cache: Some(&cache),
24320            round: None,
24321            carry_seats: false,
24322            pauses: Some((&pause, &interrupt)),
24323        };
24324        let result = wave(
24325            vec![job(
24326                sh("writer", "exit 99"),
24327                tmp.path(),
24328                Duration::from_secs(1),
24329            )],
24330            Arc::new(Semaphore::new(1)),
24331            &ctx,
24332            &mut state,
24333            0,
24334        )
24335        .await;
24336        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
24337        assert_eq!(result[0].1.turns, 0);
24338        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24339        assert_eq!(
24340            state.resource_block.unwrap().cause,
24341            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
24342        );
24343        assert!(state.active.is_empty());
24344    }
24345
24346    #[tokio::test]
24347    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
24348        crate::run::pin_test_home();
24349        let tmp = tempfile::tempdir().unwrap();
24350        let cache = tmp.path().join("target-a");
24351        let guard = hold(&cache);
24352        // Held for longer than the agent's whole 1s budget.
24353        tokio::spawn(async move {
24354            tokio::time::sleep(Duration::from_millis(1500)).await;
24355            guard.release();
24356        });
24357        let mut st = state(60);
24358        let results = run(
24359            vec![job(
24360                sh("alpha", "echo ok"),
24361                tmp.path(),
24362                Duration::from_secs(1),
24363            )],
24364            Arc::new(Semaphore::new(1)),
24365            Some(&cache),
24366            &mut st,
24367        )
24368        .await;
24369        assert!(
24370            matches!(results[0].2, AgentOutcome::Ok(_)),
24371            "the wait must not be charged to the agent"
24372        );
24373        assert!(st.resource_block.is_none());
24374        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24375        assert!(st.resource_wait.is_none() && st.active.is_empty());
24376    }
24377
24378    #[tokio::test]
24379    #[cfg(unix)]
24380    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
24381        use std::os::fd::AsRawFd as _;
24382        crate::run::pin_test_home();
24383        let tmp = tempfile::tempdir().unwrap();
24384        crate::git::git(tmp.path(), &["init"]).await.unwrap();
24385        let common = tmp.path().join(".git");
24386        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
24387        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
24388        assert_eq!(
24389            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
24390            0
24391        );
24392        let mut st = state(1);
24393        st.config.coordination = crate::config::test_coordination();
24394        let mut reader = job(
24395            sh("reader", "echo must-not-run"),
24396            tmp.path(),
24397            Duration::from_secs(30),
24398        );
24399        reader.allow_write = false;
24400        let results = run(
24401            vec![reader.clone()],
24402            Arc::new(Semaphore::new(1)),
24403            None,
24404            &mut st,
24405        )
24406        .await;
24407        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24408        assert_eq!(results[0].1.turns, 0);
24409        assert!(FailClass::of(&results[0].2).is_none());
24410        assert_eq!(
24411            st.resource_block.as_ref().unwrap().kind,
24412            "repository coordination"
24413        );
24414        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24415        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24416        drop(external);
24417        let editor = crate::repo_lock::SeatLease::for_checkout(
24418            tmp.path(),
24419            true,
24420            crate::config::test_coordination().as_ref(),
24421        )
24422        .await
24423        .unwrap();
24424        let pause = Pause::new();
24425        let interrupt = Pause::new();
24426        let prompts = Prompts::default();
24427        let ctx = WaveCtx {
24428            run: "r",
24429            node: "review",
24430            prompts: &prompts,
24431            cache: None,
24432            round: None,
24433            carry_seats: false,
24434            pauses: Some((&pause, &interrupt)),
24435        };
24436        let (results, ()) = tokio::join!(
24437            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
24438            async {
24439                tokio::time::sleep(Duration::from_millis(50)).await;
24440                pause.park();
24441            }
24442        );
24443        assert!(
24444            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
24445        );
24446        assert_eq!(results[0].1.turns, 0);
24447        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24448        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24449        drop(editor);
24450    }
24451
24452    #[tokio::test]
24453    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
24454        crate::run::pin_test_home();
24455        let tmp = tempfile::tempdir().unwrap();
24456        let cache = tmp.path().join("target-b");
24457        let _held = hold(&cache);
24458        let mut st = state(1);
24459        let results = run(
24460            vec![job(
24461                sh("alpha", "echo ok"),
24462                tmp.path(),
24463                Duration::from_secs(1),
24464            )],
24465            Arc::new(Semaphore::new(1)),
24466            Some(&cache),
24467            &mut st,
24468        )
24469        .await;
24470        let (_, seat, out) = &results[0];
24471        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
24472        assert!(wave_blocked(&results));
24473        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
24474        assert_eq!(seat.turns, 0, "nothing was asked");
24475        assert!(
24476            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
24477            "no agent was spawned"
24478        );
24479        assert!(FailClass::of(out).is_none(), "not a failure class");
24480        let block = st.resource_block.as_ref().expect("recorded for the resume");
24481        assert_eq!(block.kind, "shared build cache");
24482        assert!(matches!(
24483            block.cause,
24484            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
24485        ));
24486        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24487        assert!(st.active.is_empty());
24488    }
24489
24490    #[tokio::test]
24491    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
24492        crate::run::pin_test_home();
24493        let tmp = tempfile::tempdir().unwrap();
24494        let mut st = state(1);
24495        let results = run(
24496            vec![job(
24497                sh("alpha", "echo ok"),
24498                tmp.path(),
24499                Duration::from_secs(1),
24500            )],
24501            Arc::new(Semaphore::new(0)),
24502            None,
24503            &mut st,
24504        )
24505        .await;
24506        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24507        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
24508        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
24509    }
24510
24511    #[tokio::test]
24512    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
24513        crate::run::pin_test_home();
24514        let tmp = tempfile::tempdir().unwrap();
24515        let mut st = state(60);
24516        let results = run(
24517            vec![job(
24518                sh("alpha", "sleep 30"),
24519                tmp.path(),
24520                Duration::from_secs(1),
24521            )],
24522            Arc::new(Semaphore::new(1)),
24523            None,
24524            &mut st,
24525        )
24526        .await;
24527        let out = &results[0].2;
24528        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
24529        let class = FailClass::of(out).expect("a failure");
24530        assert_eq!(class, FailClass::Timeout);
24531        assert!(should_hand_over(None, &class), "existing handover policy");
24532        assert!(st.resource_block.is_none());
24533    }
24534}